From a768a6a41d02248ee795c941af88e283f8c81c9d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 28 May 2025 19:03:17 -0700 Subject: [PATCH] fix: introduce ResponseInputItem::McpToolCallOutput variant (#1151) The output of an MCP server tool call can be one of several types, but to date, we treated all outputs as text by showing the serialized JSON as the "tool output" in Codex: https://github.com/openai/codex/blob/25a9949c49194d5a64de54a11bcc5b4724ac9bd5/codex-rs/mcp-types/src/lib.rs#L96-L101 This PR adds support for the `ImageContent` variant so we can now display an image output from an MCP tool call. In making this change, we introduce a new `ResponseInputItem::McpToolCallOutput` variant so that we can work with the `mcp_types::CallToolResult` directly when the function call is made to an MCP server. Though arguably the more significant change is the introduction of `HistoryCell::CompletedMcpToolCallWithImageOutput`, which is a cell that uses `ratatui_image` to render an image into the terminal. To support this, we introduce `ImageRenderCache`, cache a `ratatui_image::picker::Picker`, and `ensure_image_cache()` to cache the appropriate scaled image data and dimensions based on the current terminal size. To test, I created a minimal `package.json`: ```json { "name": "kitty-mcp", "version": "1.0.0", "type": "module", "description": "MCP that returns image of kitty", "main": "index.js", "dependencies": { "@modelcontextprotocol/sdk": "^1.12.0" } } ``` with the following `index.js` to define the MCP server: ```js #!/usr/bin/env node import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; const IMAGE_URI = "image://Ada.png"; const server = new McpServer({ name: "Demo", version: "1.0.0", }); server.tool( "get-cat-image", "If you need a cat image, this tool will provide one.", async () => ({ content: [ { type: "image", data: await getAdaPngBase64(), mimeType: "image/png" }, ], }) ); server.resource("Ada the Cat", IMAGE_URI, async (uri) => { const base64Image = await getAdaPngBase64(); return { contents: [ { uri: uri.href, mimeType: "image/png", blob: base64Image, }, ], }; }); async function getAdaPngBase64() { const __dirname = new URL(".", import.meta.url).pathname; // From https://github.com/benjajaja/ratatui-image/blob/9705ce2c59ec669abbce2924cbfd1f5ae22c9860/assets/Ada.png const filePath = join(__dirname, "Ada.png"); const imageData = await readFile(filePath); const base64Image = imageData.toString("base64"); return base64Image; } const transport = new StdioServerTransport(); await server.connect(transport); ``` With the local changes from this PR, I added the following to my `config.toml`: ```toml [mcp_servers.kitty] command = "node" args = ["/Users/mbolin/code/kitty-mcp/index.js"] ``` Running the TUI from source: ``` cargo run --bin codex -- --model o3 'I need a picture of a cat' ``` I get: image Now, that said, I have only tested in iTerm and there is definitely some funny business with getting an accurate character-to-pixel ratio (sometimes the `CompletedMcpToolCallWithImageOutput` thinks it needs 10 rows to render instead of 4), so there is still work to be done here. --- codex-rs/Cargo.lock | 654 +++++++++++++++++- codex-rs/core/src/mcp_tool_call.rs | 47 +- codex-rs/core/src/models.rs | 18 + codex-rs/core/src/protocol.rs | 13 +- codex-rs/exec/src/event_processor.rs | 14 +- codex-rs/tui/Cargo.toml | 3 + codex-rs/tui/src/chatwidget.rs | 8 +- .../tui/src/conversation_history_widget.rs | 11 +- codex-rs/tui/src/history_cell.rs | 255 ++++++- 9 files changed, 936 insertions(+), 87 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 8f1762ca..97a90c15 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -54,6 +54,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" + [[package]] name = "allocative" version = "0.3.4" @@ -177,6 +183,29 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "ascii-canvas" version = "3.0.0" @@ -247,6 +276,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "av1-grain" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3efb2ca85bc610acfa917b5aaa36f3fcbebed5b3182d7f877b02531c4b80c8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98922d6a4cfbcb08820c69d8eeccc05bb1f29bfa06b4f5b1dbfe9a868bd7608e" +dependencies = [ + "arrayvec", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -304,6 +356,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -316,6 +374,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "bitstream-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" + [[package]] name = "bstr" version = "1.12.0" @@ -327,6 +391,12 @@ dependencies = [ "serde", ] +[[package]] +name = "built" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" + [[package]] name = "bumpalo" version = "3.17.0" @@ -339,12 +409,24 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" +[[package]] +name = "bytemuck" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.10.1" @@ -372,9 +454,21 @@ version = "1.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" dependencies = [ + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -536,7 +630,7 @@ dependencies = [ "path-absolutize", "predicates", "pretty_assertions", - "rand", + "rand 0.9.1", "reqwest", "seccompiler", "serde", @@ -646,6 +740,7 @@ name = "codex-tui" version = "0.0.0" dependencies = [ "anyhow", + "base64 0.22.1", "clap", "codex-ansi-escape", "codex-common", @@ -653,11 +748,13 @@ dependencies = [ "codex-linux-sandbox", "color-eyre", "crossterm", + "image", "lazy_static", "mcp-types", "path-clean", "pretty_assertions", "ratatui", + "ratatui-image", "regex", "serde_json", "shlex", @@ -700,6 +797,12 @@ dependencies = [ "tracing-error", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.3" @@ -772,6 +875,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1175,6 +1297,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide 0.8.8", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "eyre" version = "0.6.12" @@ -1202,6 +1339,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1418,6 +1564,16 @@ dependencies = [ "wasi 0.14.2+wasi-0.2.4", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.28.1" @@ -1449,6 +1605,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1641,7 +1807,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.61.0", ] [[package]] @@ -1771,6 +1937,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "icy_sixel" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc0a9c4770bc47b0a933256a496cfb8b6531f753ea9bccb19c6dff0ff7273fc" + [[package]] name = "ident_case" version = "1.0.1" @@ -1798,6 +1970,45 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b77d01e822461baa8409e156015a1d91735549f0f2c17691bd2d996bef238f7f" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" + [[package]] name = "indenter" version = "0.3.3" @@ -1845,6 +2056,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "inventory" version = "0.3.20" @@ -1886,6 +2108,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1934,6 +2165,22 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.2", + "libc", +] + +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.77" @@ -1992,12 +2239,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libredox" version = "0.1.3" @@ -2071,6 +2334,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru" version = "0.12.5" @@ -2108,6 +2380,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "mcp-types" version = "0.0.0" @@ -2169,6 +2451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -2257,6 +2540,12 @@ dependencies = [ "nom", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -2289,6 +2578,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2298,6 +2598,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2567,6 +2878,19 @@ dependencies = [ "time", ] +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.8", +] + [[package]] name = "portable-atomic" version = "1.11.0" @@ -2661,6 +2985,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" +dependencies = [ + "quote", + "syn 2.0.100", +] + [[package]] name = "pulldown-cmark" version = "0.13.0" @@ -2680,6 +3023,21 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.32.0" @@ -2714,14 +3072,35 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -2731,7 +3110,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", ] [[package]] @@ -2764,6 +3152,92 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "ratatui-image" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3f1d31464920104b247593f008158372d2fdb8165e93a4299cdd6f994448c9a" +dependencies = [ + "base64 0.21.7", + "icy_sixel", + "image", + "rand 0.8.5", + "ratatui", + "rustix 0.38.44", + "thiserror 1.0.69", + "windows", +] + +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.12.1", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand 0.8.5", + "rand_chacha 0.3.1", + "simd_helpers", + "system-deps", + "thiserror 1.0.69", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a5f31fcf7500f9401fea858ea4ab5525c99f2322cfcee732c0e6c74208c0c6" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -2911,6 +3385,12 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "rgb" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" + [[package]] name = "ring" version = "0.17.14" @@ -3348,6 +3828,21 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -3655,6 +4150,25 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempfile" version = "3.19.1" @@ -3754,6 +4268,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.41" @@ -4178,6 +4703,17 @@ dependencies = [ "serde", ] +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -4190,6 +4726,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "version_check" version = "0.9.5" @@ -4333,6 +4875,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" + [[package]] name = "wildmatch" version = "2.4.0" @@ -4370,19 +4918,53 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.0", + "windows-interface 0.59.1", "windows-link", - "windows-result", + "windows-result 0.3.2", "windows-strings 0.4.0", ] +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-implement" version = "0.60.0" @@ -4394,6 +4976,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-interface" version = "0.59.1" @@ -4417,11 +5010,20 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result", + "windows-result 0.3.2", "windows-strings 0.3.1", "windows-targets 0.53.0", ] +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.3.2" @@ -4431,6 +5033,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-strings" version = "0.3.1" @@ -4776,3 +5388,27 @@ dependencies = [ "quote", "syn 2.0.100", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99a5bab8d7dedf81405c4bb1f2b83ea057643d9cb28778cea9eecddeedd2e028" +dependencies = [ + "zune-core", +] diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 4da5b2b7..61a51a0e 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -50,51 +50,18 @@ pub(crate) async fn handle_mcp_tool_call( notify_mcp_tool_call_event(sess, sub_id, tool_call_begin_event).await; // Perform the tool call. - let (tool_call_end_event, tool_call_err) = match sess + let result = sess .call_tool(&server, &tool_name, arguments_value, timeout) .await - { - Ok(result) => ( - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success: !result.is_error.unwrap_or(false), - result: Some(result), - }), - None, - ), - Err(e) => ( - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success: false, - result: None, - }), - Some(e), - ), - }; + .map_err(|e| format!("tool call error: {e}")); + let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { + call_id: call_id.clone(), + result: result.clone(), + }); notify_mcp_tool_call_event(sess, sub_id, tool_call_end_event.clone()).await; - let EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success, - result, - }) = tool_call_end_event - else { - unimplemented!("unexpected event type"); - }; - ResponseInputItem::FunctionCallOutput { - call_id, - output: FunctionCallOutputPayload { - content: result.map_or_else( - || format!("err: {tool_call_err:?}"), - |result| { - serde_json::to_string(&result) - .unwrap_or_else(|e| format!("JSON serialization error: {e}")) - }, - ), - success: Some(success), - }, - } + ResponseInputItem::McpToolCallOutput { call_id, result } } async fn notify_mcp_tool_call_event(sess: &Session, sub_id: &str, event: EventMsg) { diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index ab213fd5..ccc550e8 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use base64::Engine; +use mcp_types::CallToolResult; use serde::Deserialize; use serde::Serialize; use serde::ser::Serializer; @@ -18,6 +19,10 @@ pub enum ResponseInputItem { call_id: String, output: FunctionCallOutputPayload, }, + McpToolCallOutput { + call_id: String, + result: Result, + }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -77,6 +82,19 @@ impl From for ResponseItem { ResponseInputItem::FunctionCallOutput { call_id, output } => { Self::FunctionCallOutput { call_id, output } } + ResponseInputItem::McpToolCallOutput { call_id, result } => Self::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + success: Some(result.is_ok()), + content: result.map_or_else( + |tool_call_err| format!("err: {tool_call_err:?}"), + |result| { + serde_json::to_string(&result) + .unwrap_or_else(|e| format!("JSON serialization error: {e}")) + }, + ), + }, + }, } } } diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 2a922cba..1b9871ed 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -396,10 +396,17 @@ pub struct McpToolCallBeginEvent { pub struct McpToolCallEndEvent { /// Identifier for the corresponding McpToolCallBegin that finished. pub call_id: String, - /// Whether the tool call was successful. If `false`, `result` might not be present. - pub success: bool, /// Result of the tool call. Note this could be an error. - pub result: Option, + pub result: Result, +} + +impl McpToolCallEndEvent { + pub fn is_success(&self) -> bool { + match &self.result { + Ok(result) => !result.is_error.unwrap_or(false), + Err(_) => false, + } + } } #[derive(Debug, Clone, Deserialize, Serialize)] diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 676b47d6..352275bf 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -242,11 +242,9 @@ impl EventProcessor { invocation.style(self.bold), ); } - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success, - result, - }) => { + EventMsg::McpToolCallEnd(tool_call_end_event) => { + let is_success = tool_call_end_event.is_success(); + let McpToolCallEndEvent { call_id, result } = tool_call_end_event; // Retrieve start time and invocation for duration calculation and labeling. let info = self.call_id_to_tool_call.remove(&call_id); @@ -261,13 +259,13 @@ impl EventProcessor { (String::new(), format!("tool('{call_id}')")) }; - let status_str = if success { "success" } else { "failed" }; - let title_style = if success { self.green } else { self.red }; + let status_str = if is_success { "success" } else { "failed" }; + let title_style = if is_success { self.green } else { self.red }; let title = format!("{invocation} {status_str}{duration}:"); ts_println!("{}", title.style(title_style)); - if let Some(res) = result { + if let Ok(res) = result { let val: serde_json::Value = res.into(); let pretty = serde_json::to_string_pretty(&val).unwrap_or_else(|_| val.to_string()); diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index c7a8361f..5886ce69 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -16,6 +16,7 @@ workspace = true [dependencies] anyhow = "1" +base64 = "0.22.1" clap = { version = "4", features = ["derive"] } codex-ansi-escape = { path = "../ansi-escape" } codex-core = { path = "../core" } @@ -23,6 +24,7 @@ codex-common = { path = "../common", features = ["cli", "elapsed"] } codex-linux-sandbox = { path = "../linux-sandbox" } color-eyre = "0.6.3" crossterm = { version = "0.28.1", features = ["bracketed-paste"] } +image = { version = "^0.25.6", default-features = false, features = ["jpeg"] } lazy_static = "1" mcp-types = { path = "../mcp-types" } path-clean = "1.0.1" @@ -30,6 +32,7 @@ ratatui = { version = "0.29.0", features = [ "unstable-widget-ref", "unstable-rendered-line-info", ] } +ratatui-image = "8.0.0" regex = "1" serde_json = "1" shlex = "1.3.0" diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 189f3994..4819be38 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -343,11 +343,9 @@ impl ChatWidget<'_> { .add_active_mcp_tool_call(call_id, server, tool, arguments); self.request_redraw(); } - EventMsg::McpToolCallEnd(McpToolCallEndEvent { - call_id, - success, - result, - }) => { + EventMsg::McpToolCallEnd(mcp_tool_call_end_event) => { + let success = mcp_tool_call_end_event.is_success(); + let McpToolCallEndEvent { call_id, result } = mcp_tool_call_end_event; self.conversation_history .record_completed_mcp_tool_call(call_id, success, result); self.request_redraw(); diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs index d69f4db8..9242e003 100644 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ b/codex-rs/tui/src/conversation_history_widget.rs @@ -293,15 +293,8 @@ impl ConversationHistoryWidget { &mut self, call_id: String, success: bool, - result: Option, + result: Result, ) { - // Convert result into serde_json::Value early so we don't have to - // worry about lifetimes inside the match arm. - let result_val = result.map(|r| { - serde_json::to_value(r) - .unwrap_or_else(|_| serde_json::Value::String("".into())) - }); - let width = self.cached_width.get(); for entry in self.entries.iter_mut() { if let HistoryCell::ActiveMcpToolCall { @@ -318,7 +311,7 @@ impl ConversationHistoryWidget { invocation.clone(), *start, success, - result_val, + result, ); entry.cell = completed; diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index c2938f4b..41c20493 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -1,24 +1,32 @@ +use crate::cell_widget::CellWidget; +use crate::exec_command::escape_command; +use crate::markdown::append_markdown; +use crate::text_block::TextBlock; +use base64::Engine; use codex_ansi_escape::ansi_escape_line; use codex_common::elapsed::format_duration; use codex_core::config::Config; use codex_core::protocol::FileChange; use codex_core::protocol::SessionConfiguredEvent; +use image::DynamicImage; +use image::GenericImageView; +use image::ImageReader; +use lazy_static::lazy_static; use ratatui::prelude::*; use ratatui::style::Color; use ratatui::style::Modifier; use ratatui::style::Style; use ratatui::text::Line as RtLine; use ratatui::text::Span as RtSpan; - -use crate::cell_widget::CellWidget; -use crate::text_block::TextBlock; +use ratatui_image::Image as TuiImage; +use ratatui_image::Resize as ImgResize; +use ratatui_image::picker::ProtocolType; use std::collections::HashMap; +use std::io::Cursor; use std::path::PathBuf; use std::time::Duration; use std::time::Instant; - -use crate::exec_command::escape_command; -use crate::markdown::append_markdown; +use tracing::error; pub(crate) struct CommandOutput { pub(crate) exit_code: i32, @@ -73,8 +81,24 @@ pub(crate) enum HistoryCell { view: TextBlock, }, - /// Completed MCP tool call. - CompletedMcpToolCall { view: TextBlock }, + /// Completed MCP tool call where we show the result serialized as JSON. + CompletedMcpToolCallWithTextOutput { view: TextBlock }, + + /// Completed MCP tool call where the result is an image. + /// Admittedly, [mcp_types::CallToolResult] can have multiple content types, + /// which could be a mix of text and images, so we need to tighten this up. + // NOTE: For image output we keep the *original* image around and lazily + // compute a resized copy that fits the available cell width. Caching the + // resized version avoids doing the potentially expensive rescale twice + // because the scroll-view first calls `height()` for layouting and then + // `render_window()` for painting. + CompletedMcpToolCallWithImageOutput { + image: DynamicImage, + /// Cached data derived from the current terminal width. The cache is + /// invalidated whenever the width changes (e.g. when the user + /// resizes the window). + render_cache: std::cell::RefCell>, + }, /// Background event. BackgroundEvent { view: TextBlock }, @@ -284,13 +308,61 @@ impl HistoryCell { } } + fn try_new_completed_mcp_tool_call_with_image_output( + result: &Result, + ) -> Option { + match result { + Ok(mcp_types::CallToolResult { content, .. }) => { + if let Some(mcp_types::CallToolResultContent::ImageContent(image)) = content.first() + { + let raw_data = + match base64::engine::general_purpose::STANDARD.decode(&image.data) { + Ok(data) => data, + Err(e) => { + error!("Failed to decode image data: {e}"); + return None; + } + }; + let reader = match ImageReader::new(Cursor::new(raw_data)).with_guessed_format() + { + Ok(reader) => reader, + Err(e) => { + error!("Failed to guess image format: {e}"); + return None; + } + }; + + let image = match reader.decode() { + Ok(image) => image, + Err(e) => { + error!("Image decoding failed: {e}"); + return None; + } + }; + + Some(HistoryCell::CompletedMcpToolCallWithImageOutput { + image, + render_cache: std::cell::RefCell::new(None), + }) + } else { + None + } + } + _ => None, + } + } + pub(crate) fn new_completed_mcp_tool_call( fq_tool_name: String, invocation: String, start: Instant, success: bool, - result: Option, + result: Result, ) -> Self { + if let Some(cell) = Self::try_new_completed_mcp_tool_call_with_image_output(&result) { + return cell; + } + let duration = format_duration(start.elapsed()); let status_str = if success { "success" } else { "failed" }; let title_line = Line::from(vec![ @@ -302,7 +374,14 @@ impl HistoryCell { lines.push(title_line); lines.push(Line::from(format!("$ {invocation}"))); - if let Some(res_val) = result { + // Convert result into serde_json::Value early so we don't have to + // worry about lifetimes inside the match arm. + let result_val = result.map(|r| { + serde_json::to_value(r) + .unwrap_or_else(|_| serde_json::Value::String("".into())) + }); + + if let Ok(res_val) = result_val { let json_pretty = serde_json::to_string_pretty(&res_val).unwrap_or_else(|_| res_val.to_string()); let mut iter = json_pretty.lines(); @@ -317,7 +396,7 @@ impl HistoryCell { lines.push(Line::from("")); - HistoryCell::CompletedMcpToolCall { + HistoryCell::CompletedMcpToolCallWithTextOutput { view: TextBlock::new(lines), } } @@ -424,10 +503,14 @@ impl CellWidget for HistoryCell { | HistoryCell::ErrorEvent { view } | HistoryCell::SessionInfo { view } | HistoryCell::CompletedExecCommand { view } - | HistoryCell::CompletedMcpToolCall { view } + | HistoryCell::CompletedMcpToolCallWithTextOutput { view } | HistoryCell::PendingPatch { view } | HistoryCell::ActiveExecCommand { view, .. } | HistoryCell::ActiveMcpToolCall { view, .. } => view.height(width), + HistoryCell::CompletedMcpToolCallWithImageOutput { + image, + render_cache, + } => ensure_image_cache(image, width, render_cache), } } @@ -441,12 +524,41 @@ impl CellWidget for HistoryCell { | HistoryCell::ErrorEvent { view } | HistoryCell::SessionInfo { view } | HistoryCell::CompletedExecCommand { view } - | HistoryCell::CompletedMcpToolCall { view } + | HistoryCell::CompletedMcpToolCallWithTextOutput { view } | HistoryCell::PendingPatch { view } | HistoryCell::ActiveExecCommand { view, .. } | HistoryCell::ActiveMcpToolCall { view, .. } => { view.render_window(first_visible_line, area, buf) } + HistoryCell::CompletedMcpToolCallWithImageOutput { + image, + render_cache, + } => { + // Ensure we have a cached, resized copy that matches the current width. + // `height()` should have prepared the cache, but if something invalidated it + // (e.g. the first `render_window()` call happens *before* `height()` after a + // resize) we rebuild it here. + + let width_cells = area.width; + + // Ensure the cache is up-to-date and extract the scaled image. + let _ = ensure_image_cache(image, width_cells, render_cache); + + let Some(resized) = render_cache + .borrow() + .as_ref() + .map(|c| c.scaled_image.clone()) + else { + return; + }; + + let picker = &*TERMINAL_PICKER; + + if let Ok(protocol) = picker.new_protocol(resized, area, ImgResize::Fit(None)) { + let img_widget = TuiImage::new(&protocol); + img_widget.render(area, buf); + } + } } } } @@ -482,3 +594,120 @@ fn create_diff_summary(changes: HashMap) -> Vec { summaries } + +// ------------------------------------- +// Helper types for image rendering +// ------------------------------------- + +/// Cached information for rendering an image inside a conversation cell. +/// +/// The cache ties the resized image to a *specific* content width (in +/// terminal cells). Whenever the terminal is resized and the width changes +/// we need to re-compute the scaled variant so that it still fits the +/// available space. Keeping the resized copy around saves a costly rescale +/// between the back-to-back `height()` and `render_window()` calls that the +/// scroll-view performs while laying out the UI. +pub(crate) struct ImageRenderCache { + /// Width in *terminal cells* the cached image was generated for. + width_cells: u16, + /// Height in *terminal rows* that the conversation cell must occupy so + /// the whole image becomes visible. + height_rows: usize, + /// The resized image that fits the given width / height constraints. + scaled_image: DynamicImage, +} + +lazy_static! { + static ref TERMINAL_PICKER: ratatui_image::picker::Picker = { + use ratatui_image::picker::Picker; + use ratatui_image::picker::cap_parser::QueryStdioOptions; + + // Ask the terminal for capabilities and explicit font size. Request the + // Kitty *text-sizing protocol* as a fallback mechanism for terminals + // (like iTerm2) that do not reply to the standard CSI 16/18 queries. + match Picker::from_query_stdio_with_options(QueryStdioOptions { + text_sizing_protocol: true, + }) { + Ok(picker) => picker, + Err(err) => { + // Fall back to the conservative default that assumes ~8×16 px cells. + // Still better than breaking the build in a headless test run. + tracing::warn!("terminal capability query failed: {err:?}; using default font size"); + Picker::from_fontsize((8, 16)) + } + } + }; +} + +/// Resize `image` to fit into `width_cells`×10-rows keeping the original aspect +/// ratio. The function updates `render_cache` and returns the number of rows +/// (<= 10) the picture will occupy. +fn ensure_image_cache( + image: &DynamicImage, + width_cells: u16, + render_cache: &std::cell::RefCell>, +) -> usize { + if let Some(cache) = render_cache.borrow().as_ref() { + if cache.width_cells == width_cells { + return cache.height_rows; + } + } + + let picker = &*TERMINAL_PICKER; + let (char_w_px, char_h_px) = picker.font_size(); + + // Heuristic to compensate for Hi-DPI terminals (iTerm2 on Retina Mac) that + // report logical pixels (≈ 8×16) while the iTerm2 graphics protocol + // expects *device* pixels. Empirically the device-pixel-ratio is almost + // always 2 on macOS Retina panels. + let hidpi_scale = if picker.protocol_type() == ProtocolType::Iterm2 { + 2.0f64 + } else { + 1.0 + }; + + // The fallback Halfblocks protocol encodes two pixel rows per cell, so each + // terminal *row* represents only half the (possibly scaled) font height. + let effective_char_h_px: f64 = if picker.protocol_type() == ProtocolType::Halfblocks { + (char_h_px as f64) * hidpi_scale / 2.0 + } else { + (char_h_px as f64) * hidpi_scale + }; + + let char_w_px_f64 = (char_w_px as f64) * hidpi_scale; + + const MAX_ROWS: f64 = 10.0; + let max_height_px: f64 = effective_char_h_px * MAX_ROWS; + + let (orig_w_px, orig_h_px) = { + let (w, h) = image.dimensions(); + (w as f64, h as f64) + }; + + if orig_w_px == 0.0 || orig_h_px == 0.0 || width_cells == 0 { + *render_cache.borrow_mut() = None; + return 0; + } + + let max_w_px = char_w_px_f64 * width_cells as f64; + let scale_w = max_w_px / orig_w_px; + let scale_h = max_height_px / orig_h_px; + let scale = scale_w.min(scale_h).min(1.0); + + use image::imageops::FilterType; + let scaled_w_px = (orig_w_px * scale).round().max(1.0) as u32; + let scaled_h_px = (orig_h_px * scale).round().max(1.0) as u32; + + let scaled_image = image.resize(scaled_w_px, scaled_h_px, FilterType::Lanczos3); + + let height_rows = ((scaled_h_px as f64 / effective_char_h_px).ceil()) as usize; + + let new_cache = ImageRenderCache { + width_cells, + height_rows, + scaled_image, + }; + *render_cache.borrow_mut() = Some(new_cache); + + height_rows +}