From 39db113cc95cf864ae07b966d07132579df05788 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 10 Sep 2025 10:18:43 -0700 Subject: [PATCH] Added images to `UserMessageEvent` (#3400) This PR adds an `images` field to the existing `UserMessageEvent` so we can encode zero or more images associated with a user message. This allows images to be restored when conversations are restored. --- codex-rs/core/src/event_mapping.rs | 115 +++++++++++++++++++++------ codex-rs/protocol/src/protocol.rs | 2 + codex-rs/tui/src/chatwidget/tests.rs | 1 + 3 files changed, 95 insertions(+), 23 deletions(-) diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index c9ad8a16..2628de42 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -25,31 +25,56 @@ pub(crate) fn map_response_item_to_event_messages( return Vec::new(); } - let events: Vec = content - .iter() - .filter_map(|content_item| match content_item { - ContentItem::OutputText { text } => { - Some(EventMsg::AgentMessage(AgentMessageEvent { - message: text.clone(), - })) - } + let mut events: Vec = Vec::new(); + let mut message_parts: Vec = Vec::new(); + let mut images: Vec = Vec::new(); + let mut kind: Option = None; + + for content_item in content.iter() { + match content_item { ContentItem::InputText { text } => { - let trimmed = text.trim_start(); - let kind = if trimmed.starts_with("") { - Some(InputMessageKind::EnvironmentContext) - } else if trimmed.starts_with("") { - Some(InputMessageKind::UserInstructions) - } else { - Some(InputMessageKind::Plain) - }; - Some(EventMsg::UserMessage(UserMessageEvent { - message: text.clone(), - kind, - })) + if kind.is_none() { + let trimmed = text.trim_start(); + kind = if trimmed.starts_with("") { + Some(InputMessageKind::EnvironmentContext) + } else if trimmed.starts_with("") { + Some(InputMessageKind::UserInstructions) + } else { + Some(InputMessageKind::Plain) + }; + } + message_parts.push(text.clone()); } - _ => None, - }) - .collect(); + ContentItem::InputImage { image_url } => { + images.push(image_url.clone()); + } + ContentItem::OutputText { text } => { + events.push(EventMsg::AgentMessage(AgentMessageEvent { + message: text.clone(), + })); + } + } + } + + if !message_parts.is_empty() || !images.is_empty() { + let message = if message_parts.is_empty() { + String::new() + } else { + message_parts.join("") + }; + let images = if images.is_empty() { + None + } else { + Some(images) + }; + + events.push(EventMsg::UserMessage(UserMessageEvent { + message, + kind, + images, + })); + } + events } @@ -96,3 +121,47 @@ pub(crate) fn map_response_item_to_event_messages( | ResponseItem::Other => Vec::new(), } } + +#[cfg(test)] +mod tests { + use super::map_response_item_to_event_messages; + use crate::protocol::EventMsg; + use crate::protocol::InputMessageKind; + use codex_protocol::models::ContentItem; + use codex_protocol::models::ResponseItem; + use pretty_assertions::assert_eq; + + #[test] + fn maps_user_message_with_text_and_two_images() { + let img1 = "https://example.com/one.png".to_string(); + let img2 = "https://example.com/two.jpg".to_string(); + + let item = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "Hello world".to_string(), + }, + ContentItem::InputImage { + image_url: img1.clone(), + }, + ContentItem::InputImage { + image_url: img2.clone(), + }, + ], + }; + + let events = map_response_item_to_event_messages(&item, false); + assert_eq!(events.len(), 1, "expected a single user message event"); + + match &events[0] { + EventMsg::UserMessage(user) => { + assert_eq!(user.message, "Hello world"); + assert!(matches!(user.kind, Some(InputMessageKind::Plain))); + assert_eq!(user.images, Some(vec![img1.clone(), img2.clone()])); + } + other => panic!("expected UserMessage, got {other:?}"), + } + } +} diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 860bb59f..7e358f12 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -695,6 +695,8 @@ pub struct UserMessageEvent { pub message: String, #[serde(skip_serializing_if = "Option::is_none")] pub kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub images: Option>, } impl From<(T, U)> for InputMessageKind diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 113864db..e097c201 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -144,6 +144,7 @@ fn resumed_initial_messages_render_history() { EventMsg::UserMessage(UserMessageEvent { message: "hello from user".to_string(), kind: Some(InputMessageKind::Plain), + images: None, }), EventMsg::AgentMessage(AgentMessageEvent { message: "assistant reply".to_string(),