From 3777e18243e51ea53f55a1a43cd658a141e7a272 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Sat, 12 Jul 2025 18:05:58 -0700 Subject: [PATCH] Add CLI streaming integration tests (#1542) ## Summary - add integration test for chat mode streaming via CLI using wiremock - add integration test for Responses API streaming via fixture - call `cargo run` to invoke the CLI during tests ## Testing - `cargo test -p codex-core --test cli_stream -- --nocapture` - `cargo clippy --all-targets --all-features -- -D warnings` ------ https://chatgpt.com/codex/tasks/task_i_68715980bbec8321999534fdd6a013c1 --- codex-rs/core/tests/cli_responses_fixture.sse | 8 ++ codex-rs/core/tests/cli_stream.rs | 119 ++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 codex-rs/core/tests/cli_responses_fixture.sse create mode 100644 codex-rs/core/tests/cli_stream.rs diff --git a/codex-rs/core/tests/cli_responses_fixture.sse b/codex-rs/core/tests/cli_responses_fixture.sse new file mode 100644 index 00000000..d297ebaf --- /dev/null +++ b/codex-rs/core/tests/cli_responses_fixture.sse @@ -0,0 +1,8 @@ +event: response.created +data: {"type":"response.created","response":{"id":"resp1"}} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture hello"}]}} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp1","output":[]}} diff --git a/codex-rs/core/tests/cli_stream.rs b/codex-rs/core/tests/cli_stream.rs new file mode 100644 index 00000000..df3fedfd --- /dev/null +++ b/codex-rs/core/tests/cli_stream.rs @@ -0,0 +1,119 @@ +#![expect(clippy::unwrap_used)] + +use assert_cmd::Command as AssertCommand; +use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; +use tempfile::TempDir; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; + +/// Tests streaming chat completions through the CLI using a mock server. +/// This test: +/// 1. Sets up a mock server that simulates OpenAI's chat completions API +/// 2. Configures codex to use this mock server via a custom provider +/// 3. Sends a simple "hello?" prompt and verifies the streamed response +/// 4. Ensures the response is received exactly once and contains "hi" +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn chat_mode_stream_cli() { + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + + let server = MockServer::start().await; + let sse = concat!( + "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{}}]}\n\n", + "data: [DONE]\n\n" + ); + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse, "text/event-stream"), + ) + .expect(1) + .mount(&server) + .await; + + let home = TempDir::new().unwrap(); + let provider_override = format!( + "model_providers.mock={{ name = \"mock\", base_url = \"{}/v1\", env_key = \"PATH\", wire_api = \"chat\" }}", + server.uri() + ); + let mut cmd = AssertCommand::new("cargo"); + cmd.arg("run") + .arg("-p") + .arg("codex-cli") + .arg("--quiet") + .arg("--") + .arg("exec") + .arg("--skip-git-repo-check") + .arg("-c") + .arg(&provider_override) + .arg("-c") + .arg("model_provider=\"mock\"") + .arg("-C") + .arg(env!("CARGO_MANIFEST_DIR")) + .arg("hello?"); + cmd.env("CODEX_HOME", home.path()) + .env("OPENAI_API_KEY", "dummy") + .env("OPENAI_BASE_URL", format!("{}/v1", server.uri())); + + let output = cmd.output().unwrap(); + println!("Status: {}", output.status); + println!("Stdout:\n{}", String::from_utf8_lossy(&output.stdout)); + println!("Stderr:\n{}", String::from_utf8_lossy(&output.stderr)); + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("hi")); + assert_eq!(stdout.matches("hi").count(), 1); + + server.verify().await; +} + +/// Tests streaming responses through the CLI using a local SSE fixture file. +/// This test: +/// 1. Uses a pre-recorded SSE response fixture instead of a live server +/// 2. Configures codex to read from this fixture via CODEX_RS_SSE_FIXTURE env var +/// 3. Sends a "hello?" prompt and verifies the response +/// 4. Ensures the fixture content is correctly streamed through the CLI +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn responses_api_stream_cli() { + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + + let fixture = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/cli_responses_fixture.sse"); + + let home = TempDir::new().unwrap(); + let mut cmd = AssertCommand::new("cargo"); + cmd.arg("run") + .arg("-p") + .arg("codex-cli") + .arg("--quiet") + .arg("--") + .arg("exec") + .arg("--skip-git-repo-check") + .arg("-C") + .arg(env!("CARGO_MANIFEST_DIR")) + .arg("hello?"); + cmd.env("CODEX_HOME", home.path()) + .env("OPENAI_API_KEY", "dummy") + .env("CODEX_RS_SSE_FIXTURE", fixture) + .env("OPENAI_BASE_URL", "http://unused.local"); + + let output = cmd.output().unwrap(); + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("fixture hello")); +}