feat: image resizing (#5446)
Add image resizing on the client side to reduce load on the API
This commit is contained in:
23
codex-rs/Cargo.lock
generated
23
codex-rs/Cargo.lock
generated
@@ -1080,6 +1080,7 @@ dependencies = [
|
|||||||
"eventsource-stream",
|
"eventsource-stream",
|
||||||
"futures",
|
"futures",
|
||||||
"http",
|
"http",
|
||||||
|
"image",
|
||||||
"indexmap 2.10.0",
|
"indexmap 2.10.0",
|
||||||
"landlock",
|
"landlock",
|
||||||
"libc",
|
"libc",
|
||||||
@@ -1335,6 +1336,7 @@ dependencies = [
|
|||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
"codex-git-tooling",
|
"codex-git-tooling",
|
||||||
|
"codex-utils-image",
|
||||||
"icu_decimal",
|
"icu_decimal",
|
||||||
"icu_locale_core",
|
"icu_locale_core",
|
||||||
"mcp-types",
|
"mcp-types",
|
||||||
@@ -1479,6 +1481,27 @@ dependencies = [
|
|||||||
"vt100",
|
"vt100",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "codex-utils-cache"
|
||||||
|
version = "0.0.0"
|
||||||
|
dependencies = [
|
||||||
|
"lru",
|
||||||
|
"sha1",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "codex-utils-image"
|
||||||
|
version = "0.0.0"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"codex-utils-cache",
|
||||||
|
"image",
|
||||||
|
"tempfile",
|
||||||
|
"thiserror 2.0.16",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "codex-utils-json-to-toml"
|
name = "codex-utils-json-to-toml"
|
||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
|
|||||||
@@ -32,9 +32,11 @@ members = [
|
|||||||
"otel",
|
"otel",
|
||||||
"tui",
|
"tui",
|
||||||
"git-apply",
|
"git-apply",
|
||||||
|
"utils/cache",
|
||||||
|
"utils/image",
|
||||||
"utils/json-to-toml",
|
"utils/json-to-toml",
|
||||||
"utils/readiness",
|
|
||||||
"utils/pty",
|
"utils/pty",
|
||||||
|
"utils/readiness",
|
||||||
"utils/string",
|
"utils/string",
|
||||||
"utils/tokenizer",
|
"utils/tokenizer",
|
||||||
]
|
]
|
||||||
@@ -77,6 +79,8 @@ codex-responses-api-proxy = { path = "responses-api-proxy" }
|
|||||||
codex-rmcp-client = { path = "rmcp-client" }
|
codex-rmcp-client = { path = "rmcp-client" }
|
||||||
codex-stdio-to-uds = { path = "stdio-to-uds" }
|
codex-stdio-to-uds = { path = "stdio-to-uds" }
|
||||||
codex-tui = { path = "tui" }
|
codex-tui = { path = "tui" }
|
||||||
|
codex-utils-cache = { path = "utils/cache" }
|
||||||
|
codex-utils-image = { path = "utils/image" }
|
||||||
codex-utils-json-to-toml = { path = "utils/json-to-toml" }
|
codex-utils-json-to-toml = { path = "utils/json-to-toml" }
|
||||||
codex-utils-pty = { path = "utils/pty" }
|
codex-utils-pty = { path = "utils/pty" }
|
||||||
codex-utils-readiness = { path = "utils/readiness" }
|
codex-utils-readiness = { path = "utils/readiness" }
|
||||||
@@ -129,6 +133,7 @@ landlock = "0.4.1"
|
|||||||
lazy_static = "1"
|
lazy_static = "1"
|
||||||
libc = "0.2.175"
|
libc = "0.2.175"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
|
lru = "0.12.5"
|
||||||
maplit = "1.0.2"
|
maplit = "1.0.2"
|
||||||
mime_guess = "2.0.5"
|
mime_guess = "2.0.5"
|
||||||
multimap = "0.10.0"
|
multimap = "0.10.0"
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ assert_cmd = { workspace = true }
|
|||||||
assert_matches = { workspace = true }
|
assert_matches = { workspace = true }
|
||||||
core_test_support = { workspace = true }
|
core_test_support = { workspace = true }
|
||||||
escargot = { workspace = true }
|
escargot = { workspace = true }
|
||||||
|
image = { workspace = true, features = ["jpeg", "png"] }
|
||||||
maplit = { workspace = true }
|
maplit = { workspace = true }
|
||||||
predicates = { workspace = true }
|
predicates = { workspace = true }
|
||||||
pretty_assertions = { workspace = true }
|
pretty_assertions = { workspace = true }
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ use core_test_support::skip_if_no_network;
|
|||||||
use core_test_support::test_codex::TestCodex;
|
use core_test_support::test_codex::TestCodex;
|
||||||
use core_test_support::test_codex::test_codex;
|
use core_test_support::test_codex::test_codex;
|
||||||
use core_test_support::wait_for_event;
|
use core_test_support::wait_for_event;
|
||||||
|
use image::GenericImageView;
|
||||||
|
use image::ImageBuffer;
|
||||||
|
use image::Rgba;
|
||||||
|
use image::load_from_memory;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use wiremock::matchers::any;
|
use wiremock::matchers::any;
|
||||||
|
|
||||||
@@ -49,6 +53,88 @@ fn extract_output_text(item: &Value) -> Option<&str> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> {
|
||||||
|
skip_if_no_network!(Ok(()));
|
||||||
|
|
||||||
|
let server = start_mock_server().await;
|
||||||
|
|
||||||
|
let TestCodex {
|
||||||
|
codex,
|
||||||
|
cwd,
|
||||||
|
session_configured,
|
||||||
|
..
|
||||||
|
} = test_codex().build(&server).await?;
|
||||||
|
|
||||||
|
let rel_path = "user-turn/example.png";
|
||||||
|
let abs_path = cwd.path().join(rel_path);
|
||||||
|
if let Some(parent) = abs_path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let image = ImageBuffer::from_pixel(4096, 1024, Rgba([20u8, 40, 60, 255]));
|
||||||
|
image.save(&abs_path)?;
|
||||||
|
|
||||||
|
let response = sse(vec![
|
||||||
|
ev_response_created("resp-1"),
|
||||||
|
ev_assistant_message("msg-1", "done"),
|
||||||
|
ev_completed("resp-1"),
|
||||||
|
]);
|
||||||
|
let mock = responses::mount_sse_once_match(&server, any(), response).await;
|
||||||
|
|
||||||
|
let session_model = session_configured.model.clone();
|
||||||
|
|
||||||
|
codex
|
||||||
|
.submit(Op::UserTurn {
|
||||||
|
items: vec![UserInput::LocalImage {
|
||||||
|
path: abs_path.clone(),
|
||||||
|
}],
|
||||||
|
final_output_json_schema: None,
|
||||||
|
cwd: cwd.path().to_path_buf(),
|
||||||
|
approval_policy: AskForApproval::Never,
|
||||||
|
sandbox_policy: SandboxPolicy::DangerFullAccess,
|
||||||
|
model: session_model,
|
||||||
|
effort: None,
|
||||||
|
summary: ReasoningSummary::Auto,
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await;
|
||||||
|
|
||||||
|
let body = mock.single_request().body_json();
|
||||||
|
let image_message =
|
||||||
|
find_image_message(&body).expect("pending input image message not included in request");
|
||||||
|
let image_url = image_message
|
||||||
|
.get("content")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.and_then(|content| {
|
||||||
|
content.iter().find_map(|span| {
|
||||||
|
if span.get("type").and_then(Value::as_str) == Some("input_image") {
|
||||||
|
span.get("image_url").and_then(Value::as_str)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.expect("image_url present");
|
||||||
|
|
||||||
|
let (prefix, encoded) = image_url
|
||||||
|
.split_once(',')
|
||||||
|
.expect("image url contains data prefix");
|
||||||
|
assert_eq!(prefix, "data:image/png;base64");
|
||||||
|
|
||||||
|
let decoded = BASE64_STANDARD
|
||||||
|
.decode(encoded)
|
||||||
|
.expect("image data decodes from base64 for request");
|
||||||
|
let resized = load_from_memory(&decoded).expect("load resized image");
|
||||||
|
let (width, height) = resized.dimensions();
|
||||||
|
assert!(width <= 2048);
|
||||||
|
assert!(height <= 768);
|
||||||
|
assert!(width < 4096);
|
||||||
|
assert!(height < 1024);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
|
async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
|
||||||
skip_if_no_network!(Ok(()));
|
skip_if_no_network!(Ok(()));
|
||||||
@@ -67,8 +153,8 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
|
|||||||
if let Some(parent) = abs_path.parent() {
|
if let Some(parent) = abs_path.parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
let image_bytes = b"fake_png_bytes".to_vec();
|
let image = ImageBuffer::from_pixel(4096, 1024, Rgba([255u8, 0, 0, 255]));
|
||||||
std::fs::write(&abs_path, &image_bytes)?;
|
image.save(&abs_path)?;
|
||||||
|
|
||||||
let call_id = "view-image-call";
|
let call_id = "view-image-call";
|
||||||
let arguments = serde_json::json!({ "path": rel_path }).to_string();
|
let arguments = serde_json::json!({ "path": rel_path }).to_string();
|
||||||
@@ -143,11 +229,20 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
|
|||||||
})
|
})
|
||||||
.expect("image_url present");
|
.expect("image_url present");
|
||||||
|
|
||||||
let expected_image_url = format!(
|
let (prefix, encoded) = image_url
|
||||||
"data:image/png;base64,{}",
|
.split_once(',')
|
||||||
BASE64_STANDARD.encode(&image_bytes)
|
.expect("image url contains data prefix");
|
||||||
);
|
assert_eq!(prefix, "data:image/png;base64");
|
||||||
assert_eq!(image_url, expected_image_url);
|
|
||||||
|
let decoded = BASE64_STANDARD
|
||||||
|
.decode(encoded)
|
||||||
|
.expect("image data decodes from base64 for request");
|
||||||
|
let resized = load_from_memory(&decoded).expect("load resized image");
|
||||||
|
let (resized_width, resized_height) = resized.dimensions();
|
||||||
|
assert!(resized_width <= 2048);
|
||||||
|
assert!(resized_height <= 768);
|
||||||
|
assert!(resized_width < 4096);
|
||||||
|
assert!(resized_height < 1024);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ workspace = true
|
|||||||
codex-git-tooling = { workspace = true }
|
codex-git-tooling = { workspace = true }
|
||||||
|
|
||||||
base64 = { workspace = true }
|
base64 = { workspace = true }
|
||||||
|
codex-utils-image = { workspace = true }
|
||||||
icu_decimal = { workspace = true }
|
icu_decimal = { workspace = true }
|
||||||
icu_locale_core = { workspace = true }
|
icu_locale_core = { workspace = true }
|
||||||
mcp-types = { workspace = true }
|
mcp-types = { workspace = true }
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
|
use codex_utils_image::load_and_resize_to_fit;
|
||||||
use mcp_types::CallToolResult;
|
use mcp_types::CallToolResult;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde::Deserializer;
|
use serde::Deserializer;
|
||||||
@@ -10,6 +11,7 @@ use ts_rs::TS;
|
|||||||
|
|
||||||
use crate::user_input::UserInput;
|
use crate::user_input::UserInput;
|
||||||
use codex_git_tooling::GhostCommit;
|
use codex_git_tooling::GhostCommit;
|
||||||
|
use codex_utils_image::error::ImageProcessingError;
|
||||||
use schemars::JsonSchema;
|
use schemars::JsonSchema;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
|
||||||
@@ -134,6 +136,19 @@ fn should_serialize_reasoning_content(content: &Option<Vec<ReasoningItemContent>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn local_image_error_placeholder(
|
||||||
|
path: &std::path::Path,
|
||||||
|
error: impl std::fmt::Display,
|
||||||
|
) -> ContentItem {
|
||||||
|
ContentItem::InputText {
|
||||||
|
text: format!(
|
||||||
|
"Codex could not read the local image at `{}`: {}",
|
||||||
|
path.display(),
|
||||||
|
error
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl From<ResponseInputItem> for ResponseItem {
|
impl From<ResponseInputItem> for ResponseItem {
|
||||||
fn from(item: ResponseInputItem) -> Self {
|
fn from(item: ResponseInputItem) -> Self {
|
||||||
match item {
|
match item {
|
||||||
@@ -217,27 +232,40 @@ impl From<Vec<UserInput>> for ResponseInputItem {
|
|||||||
role: "user".to_string(),
|
role: "user".to_string(),
|
||||||
content: items
|
content: items
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|c| match c {
|
.map(|c| match c {
|
||||||
UserInput::Text { text } => Some(ContentItem::InputText { text }),
|
UserInput::Text { text } => ContentItem::InputText { text },
|
||||||
UserInput::Image { image_url } => Some(ContentItem::InputImage { image_url }),
|
UserInput::Image { image_url } => ContentItem::InputImage { image_url },
|
||||||
UserInput::LocalImage { path } => match std::fs::read(&path) {
|
UserInput::LocalImage { path } => match load_and_resize_to_fit(&path) {
|
||||||
Ok(bytes) => {
|
Ok(image) => ContentItem::InputImage {
|
||||||
let mime = mime_guess::from_path(&path)
|
image_url: image.into_data_url(),
|
||||||
.first()
|
},
|
||||||
.map(|m| m.essence_str().to_owned())
|
|
||||||
.unwrap_or_else(|| "image".to_string());
|
|
||||||
let encoded = base64::engine::general_purpose::STANDARD.encode(bytes);
|
|
||||||
Some(ContentItem::InputImage {
|
|
||||||
image_url: format!("data:{mime};base64,{encoded}"),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
tracing::warn!(
|
tracing::warn!("Failed to resize image {}: {}", path.display(), err);
|
||||||
"Skipping image {} – could not read file: {}",
|
if matches!(&err, ImageProcessingError::Read { .. }) {
|
||||||
path.display(),
|
local_image_error_placeholder(&path, &err)
|
||||||
err
|
} else {
|
||||||
);
|
match std::fs::read(&path) {
|
||||||
None
|
Ok(bytes) => {
|
||||||
|
let mime = mime_guess::from_path(&path)
|
||||||
|
.first()
|
||||||
|
.map(|m| m.essence_str().to_owned())
|
||||||
|
.unwrap_or_else(|| "image".to_string());
|
||||||
|
let encoded =
|
||||||
|
base64::engine::general_purpose::STANDARD.encode(bytes);
|
||||||
|
ContentItem::InputImage {
|
||||||
|
image_url: format!("data:{mime};base64,{encoded}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(read_err) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"Skipping image {} – could not read file: {}",
|
||||||
|
path.display(),
|
||||||
|
read_err
|
||||||
|
);
|
||||||
|
local_image_error_placeholder(&path, &read_err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -326,6 +354,7 @@ impl std::ops::Deref for FunctionCallOutputPayload {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn serializes_success_as_plain_string() -> Result<()> {
|
fn serializes_success_as_plain_string() -> Result<()> {
|
||||||
@@ -383,4 +412,37 @@ mod tests {
|
|||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn local_image_read_error_adds_placeholder() -> Result<()> {
|
||||||
|
let dir = tempdir()?;
|
||||||
|
let missing_path = dir.path().join("missing-image.png");
|
||||||
|
|
||||||
|
let item = ResponseInputItem::from(vec![UserInput::LocalImage {
|
||||||
|
path: missing_path.clone(),
|
||||||
|
}]);
|
||||||
|
|
||||||
|
match item {
|
||||||
|
ResponseInputItem::Message { content, .. } => {
|
||||||
|
assert_eq!(content.len(), 1);
|
||||||
|
match &content[0] {
|
||||||
|
ContentItem::InputText { text } => {
|
||||||
|
let display_path = missing_path.display().to_string();
|
||||||
|
assert!(
|
||||||
|
text.contains(&display_path),
|
||||||
|
"placeholder should mention missing path: {text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
text.contains("could not read"),
|
||||||
|
"placeholder should mention read issue: {text}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
other => panic!("expected placeholder text but found {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other => panic!("expected message response but got {other:?}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
15
codex-rs/utils/cache/Cargo.toml
vendored
Normal file
15
codex-rs/utils/cache/Cargo.toml
vendored
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
[package]
|
||||||
|
name = "codex-utils-cache"
|
||||||
|
version.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
lru = { workspace = true }
|
||||||
|
sha1 = { workspace = true }
|
||||||
|
tokio = { workspace = true, features = ["sync", "rt"] }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread"] }
|
||||||
159
codex-rs/utils/cache/src/lib.rs
vendored
Normal file
159
codex-rs/utils/cache/src/lib.rs
vendored
Normal file
@@ -0,0 +1,159 @@
|
|||||||
|
use std::borrow::Borrow;
|
||||||
|
use std::hash::Hash;
|
||||||
|
use std::num::NonZeroUsize;
|
||||||
|
|
||||||
|
use lru::LruCache;
|
||||||
|
use sha1::Digest;
|
||||||
|
use sha1::Sha1;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
use tokio::sync::MutexGuard;
|
||||||
|
|
||||||
|
/// A minimal LRU cache protected by a Tokio mutex.
|
||||||
|
pub struct BlockingLruCache<K, V> {
|
||||||
|
inner: Mutex<LruCache<K, V>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<K, V> BlockingLruCache<K, V>
|
||||||
|
where
|
||||||
|
K: Eq + Hash,
|
||||||
|
{
|
||||||
|
/// Creates a cache with the provided non-zero capacity.
|
||||||
|
#[must_use]
|
||||||
|
pub fn new(capacity: NonZeroUsize) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Mutex::new(LruCache::new(capacity)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a clone of the cached value for `key`, or computes and inserts it.
|
||||||
|
pub fn get_or_insert_with(&self, key: K, value: impl FnOnce() -> V) -> V
|
||||||
|
where
|
||||||
|
V: Clone,
|
||||||
|
{
|
||||||
|
let mut guard = lock_blocking(&self.inner);
|
||||||
|
if let Some(v) = guard.get(&key) {
|
||||||
|
return v.clone();
|
||||||
|
}
|
||||||
|
let v = value();
|
||||||
|
// Insert and return a clone to keep ownership in the cache.
|
||||||
|
guard.put(key, v.clone());
|
||||||
|
v
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like `get_or_insert_with`, but the value factory may fail.
|
||||||
|
pub fn get_or_try_insert_with<E>(
|
||||||
|
&self,
|
||||||
|
key: K,
|
||||||
|
value: impl FnOnce() -> Result<V, E>,
|
||||||
|
) -> Result<V, E>
|
||||||
|
where
|
||||||
|
V: Clone,
|
||||||
|
{
|
||||||
|
let mut guard = lock_blocking(&self.inner);
|
||||||
|
if let Some(v) = guard.get(&key) {
|
||||||
|
return Ok(v.clone());
|
||||||
|
}
|
||||||
|
let v = value()?;
|
||||||
|
guard.put(key, v.clone());
|
||||||
|
Ok(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds a cache if `capacity` is non-zero, returning `None` otherwise.
|
||||||
|
#[must_use]
|
||||||
|
pub fn try_with_capacity(capacity: usize) -> Option<Self> {
|
||||||
|
NonZeroUsize::new(capacity).map(Self::new)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a clone of the cached value corresponding to `key`, if present.
|
||||||
|
pub fn get<Q>(&self, key: &Q) -> Option<V>
|
||||||
|
where
|
||||||
|
K: Borrow<Q>,
|
||||||
|
Q: Hash + Eq + ?Sized,
|
||||||
|
V: Clone,
|
||||||
|
{
|
||||||
|
lock_blocking(&self.inner).get(key).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Inserts `value` for `key`, returning the previous entry if it existed.
|
||||||
|
pub fn insert(&self, key: K, value: V) -> Option<V> {
|
||||||
|
lock_blocking(&self.inner).put(key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Removes the entry for `key` if it exists, returning it.
|
||||||
|
pub fn remove<Q>(&self, key: &Q) -> Option<V>
|
||||||
|
where
|
||||||
|
K: Borrow<Q>,
|
||||||
|
Q: Hash + Eq + ?Sized,
|
||||||
|
{
|
||||||
|
lock_blocking(&self.inner).pop(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clears all entries from the cache.
|
||||||
|
pub fn clear(&self) {
|
||||||
|
lock_blocking(&self.inner).clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Executes `callback` with a mutable reference to the underlying cache.
|
||||||
|
pub fn with_mut<R>(&self, callback: impl FnOnce(&mut LruCache<K, V>) -> R) -> R {
|
||||||
|
let mut guard = lock_blocking(&self.inner);
|
||||||
|
callback(&mut guard)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Provides direct access to the cache guard for advanced use cases.
|
||||||
|
pub fn blocking_lock(&self) -> MutexGuard<'_, LruCache<K, V>> {
|
||||||
|
lock_blocking(&self.inner)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lock_blocking<K, V>(m: &Mutex<LruCache<K, V>>) -> MutexGuard<'_, LruCache<K, V>>
|
||||||
|
where
|
||||||
|
K: Eq + Hash,
|
||||||
|
{
|
||||||
|
match tokio::runtime::Handle::try_current() {
|
||||||
|
Ok(_) => tokio::task::block_in_place(|| m.blocking_lock()),
|
||||||
|
Err(_) => m.blocking_lock(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Computes the SHA-1 digest of `bytes`.
|
||||||
|
///
|
||||||
|
/// Useful for content-based cache keys when you want to avoid staleness
|
||||||
|
/// caused by path-only keys.
|
||||||
|
#[must_use]
|
||||||
|
pub fn sha1_digest(bytes: &[u8]) -> [u8; 20] {
|
||||||
|
let mut hasher = Sha1::new();
|
||||||
|
hasher.update(bytes);
|
||||||
|
let result = hasher.finalize();
|
||||||
|
let mut out = [0; 20];
|
||||||
|
out.copy_from_slice(&result);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::BlockingLruCache;
|
||||||
|
use std::num::NonZeroUsize;
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn stores_and_retrieves_values() {
|
||||||
|
let cache = BlockingLruCache::new(NonZeroUsize::new(2).expect("capacity"));
|
||||||
|
|
||||||
|
assert!(cache.get(&"first").is_none());
|
||||||
|
cache.insert("first", 1);
|
||||||
|
assert_eq!(cache.get(&"first"), Some(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn evicts_least_recently_used() {
|
||||||
|
let cache = BlockingLruCache::new(NonZeroUsize::new(2).expect("capacity"));
|
||||||
|
cache.insert("a", 1);
|
||||||
|
cache.insert("b", 2);
|
||||||
|
assert_eq!(cache.get(&"a"), Some(1));
|
||||||
|
|
||||||
|
cache.insert("c", 3);
|
||||||
|
|
||||||
|
assert!(cache.get(&"b").is_none());
|
||||||
|
assert_eq!(cache.get(&"a"), Some(1));
|
||||||
|
assert_eq!(cache.get(&"c"), Some(3));
|
||||||
|
}
|
||||||
|
}
|
||||||
18
codex-rs/utils/image/Cargo.toml
Normal file
18
codex-rs/utils/image/Cargo.toml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
[package]
|
||||||
|
name = "codex-utils-image"
|
||||||
|
version.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
base64 = { workspace = true }
|
||||||
|
image = { workspace = true, features = ["jpeg", "png"] }
|
||||||
|
codex-utils-cache = { workspace = true }
|
||||||
|
thiserror = { workspace = true }
|
||||||
|
tokio = { workspace = true, features = ["fs", "rt", "rt-multi-thread", "macros"] }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
image = { workspace = true, features = ["jpeg", "png"] }
|
||||||
|
tempfile = { workspace = true }
|
||||||
25
codex-rs/utils/image/src/error.rs
Normal file
25
codex-rs/utils/image/src/error.rs
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
use image::ImageFormat;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use thiserror::Error;
|
||||||
|
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum ImageProcessingError {
|
||||||
|
#[error("failed to read image at {path}: {source}")]
|
||||||
|
Read {
|
||||||
|
path: PathBuf,
|
||||||
|
#[source]
|
||||||
|
source: std::io::Error,
|
||||||
|
},
|
||||||
|
#[error("failed to decode image at {path}: {source}")]
|
||||||
|
Decode {
|
||||||
|
path: PathBuf,
|
||||||
|
#[source]
|
||||||
|
source: image::ImageError,
|
||||||
|
},
|
||||||
|
#[error("failed to encode image as {format:?}: {source}")]
|
||||||
|
Encode {
|
||||||
|
format: ImageFormat,
|
||||||
|
#[source]
|
||||||
|
source: image::ImageError,
|
||||||
|
},
|
||||||
|
}
|
||||||
252
codex-rs/utils/image/src/lib.rs
Normal file
252
codex-rs/utils/image/src/lib.rs
Normal file
@@ -0,0 +1,252 @@
|
|||||||
|
use std::num::NonZeroUsize;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
|
use crate::error::ImageProcessingError;
|
||||||
|
use base64::Engine;
|
||||||
|
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||||
|
use codex_utils_cache::BlockingLruCache;
|
||||||
|
use codex_utils_cache::sha1_digest;
|
||||||
|
use image::ColorType;
|
||||||
|
use image::DynamicImage;
|
||||||
|
use image::GenericImageView;
|
||||||
|
use image::ImageEncoder;
|
||||||
|
use image::ImageFormat;
|
||||||
|
use image::codecs::jpeg::JpegEncoder;
|
||||||
|
use image::codecs::png::PngEncoder;
|
||||||
|
use image::imageops::FilterType;
|
||||||
|
/// Maximum width used when resizing images before uploading.
|
||||||
|
pub const MAX_WIDTH: u32 = 2048;
|
||||||
|
/// Maximum height used when resizing images before uploading.
|
||||||
|
pub const MAX_HEIGHT: u32 = 768;
|
||||||
|
|
||||||
|
pub mod error;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct EncodedImage {
|
||||||
|
pub bytes: Vec<u8>,
|
||||||
|
pub mime: String,
|
||||||
|
pub width: u32,
|
||||||
|
pub height: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EncodedImage {
|
||||||
|
pub fn into_data_url(self) -> String {
|
||||||
|
let encoded = BASE64_STANDARD.encode(&self.bytes);
|
||||||
|
format!("data:{};base64,{}", self.mime, encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static IMAGE_CACHE: LazyLock<BlockingLruCache<[u8; 20], EncodedImage>> =
|
||||||
|
LazyLock::new(|| BlockingLruCache::new(NonZeroUsize::new(32).unwrap_or(NonZeroUsize::MIN)));
|
||||||
|
|
||||||
|
pub fn load_and_resize_to_fit(path: &Path) -> Result<EncodedImage, ImageProcessingError> {
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
|
||||||
|
let file_bytes = read_file_bytes(path, &path_buf)?;
|
||||||
|
|
||||||
|
let key = sha1_digest(&file_bytes);
|
||||||
|
|
||||||
|
IMAGE_CACHE.get_or_try_insert_with(key, move || {
|
||||||
|
let format = match image::guess_format(&file_bytes) {
|
||||||
|
Ok(ImageFormat::Png) => Some(ImageFormat::Png),
|
||||||
|
Ok(ImageFormat::Jpeg) => Some(ImageFormat::Jpeg),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let dynamic = image::load_from_memory(&file_bytes).map_err(|source| {
|
||||||
|
ImageProcessingError::Decode {
|
||||||
|
path: path_buf.clone(),
|
||||||
|
source,
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let (width, height) = dynamic.dimensions();
|
||||||
|
|
||||||
|
let encoded = if width <= MAX_WIDTH && height <= MAX_HEIGHT {
|
||||||
|
if let Some(format) = format {
|
||||||
|
let mime = format_to_mime(format);
|
||||||
|
EncodedImage {
|
||||||
|
bytes: file_bytes,
|
||||||
|
mime,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png)?;
|
||||||
|
let mime = format_to_mime(output_format);
|
||||||
|
EncodedImage {
|
||||||
|
bytes,
|
||||||
|
mime,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let resized = dynamic.resize(MAX_WIDTH, MAX_HEIGHT, FilterType::Triangle);
|
||||||
|
let target_format = format.unwrap_or(ImageFormat::Png);
|
||||||
|
let (bytes, output_format) = encode_image(&resized, target_format)?;
|
||||||
|
let mime = format_to_mime(output_format);
|
||||||
|
EncodedImage {
|
||||||
|
bytes,
|
||||||
|
mime,
|
||||||
|
width: resized.width(),
|
||||||
|
height: resized.height(),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(encoded)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_file_bytes(path: &Path, path_for_error: &Path) -> Result<Vec<u8>, ImageProcessingError> {
|
||||||
|
match tokio::runtime::Handle::try_current() {
|
||||||
|
// If we're inside a Tokio runtime, avoid block_on (it panics on worker threads).
|
||||||
|
// Use block_in_place and do a standard blocking read safely.
|
||||||
|
Ok(_) => tokio::task::block_in_place(|| std::fs::read(path)).map_err(|source| {
|
||||||
|
ImageProcessingError::Read {
|
||||||
|
path: path_for_error.to_path_buf(),
|
||||||
|
source,
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
// Outside a runtime, just read synchronously.
|
||||||
|
Err(_) => std::fs::read(path).map_err(|source| ImageProcessingError::Read {
|
||||||
|
path: path_for_error.to_path_buf(),
|
||||||
|
source,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_image(
|
||||||
|
image: &DynamicImage,
|
||||||
|
preferred_format: ImageFormat,
|
||||||
|
) -> Result<(Vec<u8>, ImageFormat), ImageProcessingError> {
|
||||||
|
let target_format = match preferred_format {
|
||||||
|
ImageFormat::Jpeg => ImageFormat::Jpeg,
|
||||||
|
_ => ImageFormat::Png,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
|
||||||
|
match target_format {
|
||||||
|
ImageFormat::Png => {
|
||||||
|
let rgba = image.to_rgba8();
|
||||||
|
let encoder = PngEncoder::new(&mut buffer);
|
||||||
|
encoder
|
||||||
|
.write_image(
|
||||||
|
rgba.as_raw(),
|
||||||
|
image.width(),
|
||||||
|
image.height(),
|
||||||
|
ColorType::Rgba8.into(),
|
||||||
|
)
|
||||||
|
.map_err(|source| ImageProcessingError::Encode {
|
||||||
|
format: target_format,
|
||||||
|
source,
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
ImageFormat::Jpeg => {
|
||||||
|
let mut encoder = JpegEncoder::new_with_quality(&mut buffer, 85);
|
||||||
|
encoder
|
||||||
|
.encode_image(image)
|
||||||
|
.map_err(|source| ImageProcessingError::Encode {
|
||||||
|
format: target_format,
|
||||||
|
source,
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
_ => unreachable!("unsupported target_format should have been handled earlier"),
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok((buffer, target_format))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_to_mime(format: ImageFormat) -> String {
|
||||||
|
match format {
|
||||||
|
ImageFormat::Jpeg => "image/jpeg".to_string(),
|
||||||
|
_ => "image/png".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use image::GenericImageView;
|
||||||
|
use image::ImageBuffer;
|
||||||
|
use image::Rgba;
|
||||||
|
use tempfile::NamedTempFile;
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn returns_original_image_when_within_bounds() {
|
||||||
|
let temp_file = NamedTempFile::new().expect("temp file");
|
||||||
|
let image = ImageBuffer::from_pixel(64, 32, Rgba([10u8, 20, 30, 255]));
|
||||||
|
image
|
||||||
|
.save_with_format(temp_file.path(), ImageFormat::Png)
|
||||||
|
.expect("write png to temp file");
|
||||||
|
|
||||||
|
let original_bytes = std::fs::read(temp_file.path()).expect("read written image");
|
||||||
|
|
||||||
|
let encoded = load_and_resize_to_fit(temp_file.path()).expect("process image");
|
||||||
|
|
||||||
|
assert_eq!(encoded.width, 64);
|
||||||
|
assert_eq!(encoded.height, 32);
|
||||||
|
assert_eq!(encoded.mime, "image/png");
|
||||||
|
assert_eq!(encoded.bytes, original_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn downscales_large_image() {
|
||||||
|
let temp_file = NamedTempFile::new().expect("temp file");
|
||||||
|
let image = ImageBuffer::from_pixel(4096, 2048, Rgba([200u8, 10, 10, 255]));
|
||||||
|
image
|
||||||
|
.save_with_format(temp_file.path(), ImageFormat::Png)
|
||||||
|
.expect("write png to temp file");
|
||||||
|
|
||||||
|
let processed = load_and_resize_to_fit(temp_file.path()).expect("process image");
|
||||||
|
|
||||||
|
assert!(processed.width <= MAX_WIDTH);
|
||||||
|
assert!(processed.height <= MAX_HEIGHT);
|
||||||
|
|
||||||
|
let loaded =
|
||||||
|
image::load_from_memory(&processed.bytes).expect("read resized bytes back into image");
|
||||||
|
assert_eq!(loaded.dimensions(), (processed.width, processed.height));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn fails_cleanly_for_invalid_images() {
|
||||||
|
let temp_file = NamedTempFile::new().expect("temp file");
|
||||||
|
std::fs::write(temp_file.path(), b"not an image").expect("write bytes");
|
||||||
|
|
||||||
|
let err = load_and_resize_to_fit(temp_file.path()).expect_err("invalid image should fail");
|
||||||
|
match err {
|
||||||
|
ImageProcessingError::Decode { .. } => {}
|
||||||
|
_ => panic!("unexpected error variant"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn reprocesses_updated_file_contents() {
|
||||||
|
{
|
||||||
|
IMAGE_CACHE.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
let temp_file = NamedTempFile::new().expect("temp file");
|
||||||
|
let first_image = ImageBuffer::from_pixel(32, 16, Rgba([20u8, 120, 220, 255]));
|
||||||
|
first_image
|
||||||
|
.save_with_format(temp_file.path(), ImageFormat::Png)
|
||||||
|
.expect("write initial image");
|
||||||
|
|
||||||
|
let first = load_and_resize_to_fit(temp_file.path()).expect("process first image");
|
||||||
|
|
||||||
|
let second_image = ImageBuffer::from_pixel(96, 48, Rgba([50u8, 60, 70, 255]));
|
||||||
|
second_image
|
||||||
|
.save_with_format(temp_file.path(), ImageFormat::Png)
|
||||||
|
.expect("write updated image");
|
||||||
|
|
||||||
|
let second = load_and_resize_to_fit(temp_file.path()).expect("process updated image");
|
||||||
|
|
||||||
|
assert_eq!(first.width, 32);
|
||||||
|
assert_eq!(first.height, 16);
|
||||||
|
assert_eq!(second.width, 96);
|
||||||
|
assert_eq!(second.height, 48);
|
||||||
|
assert_ne!(second.bytes, first.bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user