OpenTelemetry events (#2103)

### Title

## otel

Codex can emit [OpenTelemetry](https://opentelemetry.io/) **log events**
that
describe each run: outbound API requests, streamed responses, user
input,
tool-approval decisions, and the result of every tool invocation. Export
is
**disabled by default** so local runs remain self-contained. Opt in by
adding an
`[otel]` table and choosing an exporter.

```toml
[otel]
environment = "staging"   # defaults to "dev"
exporter = "none"          # defaults to "none"; set to otlp-http or otlp-grpc to send events
log_user_prompt = false    # defaults to false; redact prompt text unless explicitly enabled
```

Codex tags every exported event with `service.name = "codex-cli"`, the
CLI
version, and an `env` attribute so downstream collectors can distinguish
dev/staging/prod traffic. Only telemetry produced inside the
`codex_otel`
crate—the events listed below—is forwarded to the exporter.

### Event catalog

Every event shares a common set of metadata fields: `event.timestamp`,
`conversation.id`, `app.version`, `auth_mode` (when available),
`user.account_id` (when available), `terminal.type`, `model`, and
`slug`.

With OTEL enabled Codex emits the following event types (in addition to
the
metadata above):

- `codex.api_request`
  - `cf_ray` (optional)
  - `attempt`
  - `duration_ms`
  - `http.response.status_code` (optional)
  - `error.message` (failures)
- `codex.sse_event`
  - `event.kind`
  - `duration_ms`
  - `error.message` (failures)
  - `input_token_count` (completion only)
  - `output_token_count` (completion only)
  - `cached_token_count` (completion only, optional)
  - `reasoning_token_count` (completion only, optional)
  - `tool_token_count` (completion only)
- `codex.user_prompt`
  - `prompt_length`
  - `prompt` (redacted unless `log_user_prompt = true`)
- `codex.tool_decision`
  - `tool_name`
  - `call_id`
- `decision` (`approved`, `approved_for_session`, `denied`, or `abort`)
  - `source` (`config` or `user`)
- `codex.tool_result`
  - `tool_name`
  - `call_id`
  - `arguments`
  - `duration_ms` (execution time for the tool)
  - `success` (`"true"` or `"false"`)
  - `output`

### Choosing an exporter

Set `otel.exporter` to control where events go:

- `none` – leaves instrumentation active but skips exporting. This is
the
  default.
- `otlp-http` – posts OTLP log records to an OTLP/HTTP collector.
Specify the
  endpoint, protocol, and headers your collector expects:

  ```toml
  [otel]
  exporter = { otlp-http = {
    endpoint = "https://otel.example.com/v1/logs",
    protocol = "binary",
    headers = { "x-otlp-api-key" = "${OTLP_TOKEN}" }
  }}
  ```

- `otlp-grpc` – streams OTLP log records over gRPC. Provide the endpoint
and any
  metadata headers:

  ```toml
  [otel]
  exporter = { otlp-grpc = {
    endpoint = "https://otel.example.com:4317",
    headers = { "x-otlp-meta" = "abc123" }
  }}
  ```

If the exporter is `none` nothing is written anywhere; otherwise you
must run or point to your
own collector. All exporters run on a background batch worker that is
flushed on
shutdown.

If you build Codex from source the OTEL crate is still behind an `otel`
feature
flag; the official prebuilt binaries ship with the feature enabled. When
the
feature is disabled the telemetry hooks become no-ops so the CLI
continues to
function without the extra dependencies.

---------

Co-authored-by: Anton Panasenko <apanasenko@openai.com>

codex-rs/core/tests/chat_completions_payload.rs

@@ -11,6 +11,8 @@ use codex_core::ReasoningItemContent;
 use codex_core::ResponseItem;
 use codex_core::WireApi;
 use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use codex_otel::otel_event_manager::OtelEventManager;
+use codex_protocol::mcp_protocol::AuthMode;
 use codex_protocol::mcp_protocol::ConversationId;
 use core_test_support::load_default_config_for_test;
 use futures::StreamExt;
@@ -70,13 +72,26 @@ async fn run_request(input: Vec<ResponseItem>) -> Value {
     let summary = config.model_reasoning_summary;
     let config = Arc::new(config);
 
+    let conversation_id = ConversationId::new();
+
+    let otel_event_manager = OtelEventManager::new(
+        conversation_id,
+        config.model.as_str(),
+        config.model_family.slug.as_str(),
+        None,
+        Some(AuthMode::ChatGPT),
+        false,
+        "test".to_string(),
+    );
+
     let client = ModelClient::new(
         Arc::clone(&config),
         None,
+        otel_event_manager,
         provider,
         effort,
         summary,
-        ConversationId::new(),
+        conversation_id,
     );
 
     let mut prompt = Prompt::default();

codex-rs/core/tests/chat_completions_sse.rs

@@ -1,4 +1,5 @@
 use std::sync::Arc;
+use tracing_test::traced_test;
 
 use codex_core::ContentItem;
 use codex_core::ModelClient;
@@ -8,6 +9,8 @@ use codex_core::ResponseEvent;
 use codex_core::ResponseItem;
 use codex_core::WireApi;
 use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use codex_otel::otel_event_manager::OtelEventManager;
+use codex_protocol::mcp_protocol::AuthMode;
 use codex_protocol::mcp_protocol::ConversationId;
 use core_test_support::load_default_config_for_test;
 use futures::StreamExt;
@@ -23,11 +26,15 @@ fn network_disabled() -> bool {
 }
 
 async fn run_stream(sse_body: &str) -> Vec<ResponseEvent> {
+    run_stream_with_bytes(sse_body.as_bytes()).await
+}
+
+async fn run_stream_with_bytes(sse_body: &[u8]) -> Vec<ResponseEvent> {
     let server = MockServer::start().await;
 
     let template = ResponseTemplate::new(200)
         .insert_header("content-type", "text/event-stream")
-        .set_body_raw(sse_body.to_string(), "text/event-stream");
+        .set_body_bytes(sse_body.to_vec());
 
     Mock::given(method("POST"))
         .and(path("/v1/chat/completions"))
@@ -63,13 +70,26 @@ async fn run_stream(sse_body: &str) -> Vec<ResponseEvent> {
     let summary = config.model_reasoning_summary;
     let config = Arc::new(config);
 
+    let conversation_id = ConversationId::new();
+
+    let otel_event_manager = OtelEventManager::new(
+        conversation_id,
+        config.model.as_str(),
+        config.model_family.slug.as_str(),
+        None,
+        Some(AuthMode::ChatGPT),
+        false,
+        "test".to_string(),
+    );
+
     let client = ModelClient::new(
         Arc::clone(&config),
         None,
+        otel_event_manager,
         provider,
         effort,
         summary,
-        ConversationId::new(),
+        conversation_id,
     );
 
     let mut prompt = Prompt::default();
@@ -89,7 +109,8 @@ async fn run_stream(sse_body: &str) -> Vec<ResponseEvent> {
     while let Some(event) = stream.next().await {
         match event {
             Ok(ev) => events.push(ev),
-            Err(e) => panic!("stream event error: {e}"),
+            // We still collect the error to exercise telemetry and complete the task.
+            Err(_e) => break,
         }
     }
     events
@@ -318,3 +339,88 @@ async fn streams_reasoning_before_tool_call() {
 
     assert!(matches!(events[3], ResponseEvent::Completed { .. }));
 }
+
+#[tokio::test]
+#[traced_test]
+async fn chat_sse_emits_failed_on_parse_error() {
+    if network_disabled() {
+        println!(
+            "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+        );
+        return;
+    }
+
+    let sse_body = concat!("data: not-json\n\n", "data: [DONE]\n\n");
+
+    let _ = run_stream(sse_body).await;
+
+    logs_assert(|lines: &[&str]| {
+        lines
+            .iter()
+            .find(|line| {
+                line.contains("codex.api_request") && line.contains("http.response.status_code=200")
+            })
+            .map(|_| Ok(()))
+            .unwrap_or(Err("cannot find codex.api_request event".to_string()))
+    });
+
+    logs_assert(|lines: &[&str]| {
+        lines
+            .iter()
+            .find(|line| {
+                line.contains("codex.sse_event")
+                    && line.contains("error.message")
+                    && line.contains("expected ident at line 1 column 2")
+            })
+            .map(|_| Ok(()))
+            .unwrap_or(Err("cannot find SSE event".to_string()))
+    });
+}
+
+#[tokio::test]
+#[traced_test]
+async fn chat_sse_done_chunk_emits_event() {
+    if network_disabled() {
+        println!(
+            "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+        );
+        return;
+    }
+
+    let sse_body = "data: [DONE]\n\n";
+
+    let _ = run_stream(sse_body).await;
+
+    logs_assert(|lines: &[&str]| {
+        lines
+            .iter()
+            .find(|line| line.contains("codex.sse_event") && line.contains("event.kind=message"))
+            .map(|_| Ok(()))
+            .unwrap_or(Err("cannot find SSE event".to_string()))
+    });
+}
+
+#[tokio::test]
+#[traced_test]
+async fn chat_sse_emits_error_on_invalid_utf8() {
+    if network_disabled() {
+        println!(
+            "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+        );
+        return;
+    }
+
+    let _ = run_stream_with_bytes(b"data: \x80\x80\n\n").await;
+
+    logs_assert(|lines: &[&str]| {
+        lines
+            .iter()
+            .find(|line| {
+                line.contains("codex.sse_event")
+                    && line.contains("error.message")
+                    && line.contains("UTF8 error: invalid utf-8 sequence of 1 bytes from index 0")
+            })
+            .map(|_| Ok(()))
+            .unwrap_or(Err("cannot find SSE event".to_string()))
+    });
+}

codex-rs/core/tests/common/responses.rs

@@ -75,6 +75,33 @@ pub fn ev_function_call(call_id: &str, name: &str, arguments: &str) -> Value {
     })
 }
 
+pub fn ev_custom_tool_call(call_id: &str, name: &str, input: &str) -> Value {
+    serde_json::json!({
+        "type": "response.output_item.done",
+        "item": {
+            "type": "custom_tool_call",
+            "call_id": call_id,
+            "name": name,
+            "input": input
+        }
+    })
+}
+
+pub fn ev_local_shell_call(call_id: &str, status: &str, command: Vec<&str>) -> Value {
+    serde_json::json!({
+        "type": "response.output_item.done",
+        "item": {
+            "type": "local_shell_call",
+            "call_id": call_id,
+            "status": status,
+            "action": {
+                "type": "exec",
+                "command": command,
+            }
+        }
+    })
+}
+
 /// Convenience: SSE event for an `apply_patch` custom tool call with raw patch
 /// text. This mirrors the payload produced by the Responses API when the model
 /// invokes `apply_patch` directly (before we convert it to a function call).
@@ -114,7 +141,7 @@ pub fn sse_response(body: String) -> ResponseTemplate {
         .set_body_raw(body, "text/event-stream")
 }
 
-pub async fn mount_sse_once<M>(server: &MockServer, matcher: M, body: String)
+pub async fn mount_sse_once_match<M>(server: &MockServer, matcher: M, body: String)
 where
     M: wiremock::Match + Send + Sync + 'static,
 {
@@ -127,6 +154,23 @@ where
         .await;
 }
 
+pub async fn mount_sse_once(server: &MockServer, body: String) {
+    Mock::given(method("POST"))
+        .and(path("/v1/responses"))
+        .respond_with(sse_response(body))
+        .expect(1)
+        .mount(server)
+        .await;
+}
+
+pub async fn mount_sse(server: &MockServer, body: String) {
+    Mock::given(method("POST"))
+        .and(path("/v1/responses"))
+        .respond_with(sse_response(body))
+        .mount(server)
+        .await;
+}
+
 pub async fn start_mock_server() -> MockServer {
     MockServer::builder()
         .body_print_limit(BodyPrintLimit::Limited(80_000))

codex-rs/core/tests/suite/abort_tasks.rs

@@ -4,7 +4,7 @@ use codex_core::protocol::EventMsg;
 use codex_core::protocol::InputItem;
 use codex_core::protocol::Op;
 use core_test_support::responses::ev_function_call;
-use core_test_support::responses::mount_sse_once;
+use core_test_support::responses::mount_sse_once_match;
 use core_test_support::responses::sse;
 use core_test_support::responses::start_mock_server;
 use core_test_support::test_codex::test_codex;
@@ -30,7 +30,7 @@ async fn interrupt_long_running_tool_emits_turn_aborted() {
     let body = sse(vec![ev_function_call("call_sleep", "shell", &args)]);
 
     let server = start_mock_server().await;
-    mount_sse_once(&server, body_string_contains("start sleep"), body).await;
+    mount_sse_once_match(&server, body_string_contains("start sleep"), body).await;
 
     let codex = test_codex().build(&server).await.unwrap().codex;
 

codex-rs/core/tests/suite/client.rs

@@ -16,6 +16,8 @@ use codex_core::built_in_model_providers;
 use codex_core::protocol::EventMsg;
 use codex_core::protocol::InputItem;
 use codex_core::protocol::Op;
+use codex_otel::otel_event_manager::OtelEventManager;
+use codex_protocol::mcp_protocol::AuthMode;
 use codex_protocol::mcp_protocol::ConversationId;
 use codex_protocol::models::ReasoningItemReasoningSummary;
 use codex_protocol::models::WebSearchAction;
@@ -664,13 +666,26 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() {
     let summary = config.model_reasoning_summary;
     let config = Arc::new(config);
 
+    let conversation_id = ConversationId::new();
+
+    let otel_event_manager = OtelEventManager::new(
+        conversation_id,
+        config.model.as_str(),
+        config.model_family.slug.as_str(),
+        None,
+        Some(AuthMode::ChatGPT),
+        false,
+        "test".to_string(),
+    );
+
     let client = ModelClient::new(
         Arc::clone(&config),
         None,
+        otel_event_manager,
         provider,
         effort,
         summary,
-        ConversationId::new(),
+        conversation_id,
     );
 
     let mut prompt = Prompt::default();

codex-rs/core/tests/suite/compact.rs

@@ -25,7 +25,7 @@ use core_test_support::responses::ev_assistant_message;
 use core_test_support::responses::ev_completed;
 use core_test_support::responses::ev_completed_with_tokens;
 use core_test_support::responses::ev_function_call;
-use core_test_support::responses::mount_sse_once;
+use core_test_support::responses::mount_sse_once_match;
 use core_test_support::responses::sse;
 use core_test_support::responses::sse_response;
 use core_test_support::responses::start_mock_server;
@@ -79,19 +79,19 @@ async fn summarize_context_three_requests_and_instructions() {
         body.contains("\"text\":\"hello world\"")
             && !body.contains("You have exceeded the maximum number of tokens")
     };
-    mount_sse_once(&server, first_matcher, sse1).await;
+    mount_sse_once_match(&server, first_matcher, sse1).await;
 
     let second_matcher = |req: &wiremock::Request| {
         let body = std::str::from_utf8(&req.body).unwrap_or("");
         body.contains("You have exceeded the maximum number of tokens")
     };
-    mount_sse_once(&server, second_matcher, sse2).await;
+    mount_sse_once_match(&server, second_matcher, sse2).await;
 
     let third_matcher = |req: &wiremock::Request| {
         let body = std::str::from_utf8(&req.body).unwrap_or("");
         body.contains(&format!("\"text\":\"{THIRD_USER_MSG}\""))
     };
-    mount_sse_once(&server, third_matcher, sse3).await;
+    mount_sse_once_match(&server, third_matcher, sse3).await;
 
     // Build config pointing to the mock server and spawn Codex.
     let model_provider = ModelProviderInfo {

codex-rs/core/tests/suite/compact_resume_fork.rs

@@ -25,7 +25,7 @@ use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
 use core_test_support::load_default_config_for_test;
 use core_test_support::responses::ev_assistant_message;
 use core_test_support::responses::ev_completed;
-use core_test_support::responses::mount_sse_once;
+use core_test_support::responses::mount_sse_once_match;
 use core_test_support::responses::sse;
 use core_test_support::wait_for_event;
 use pretty_assertions::assert_eq;
@@ -702,13 +702,13 @@ async fn mount_initial_flow(server: &MockServer) {
             && !body.contains("\"text\":\"AFTER_RESUME\"")
             && !body.contains("\"text\":\"AFTER_FORK\"")
     };
-    mount_sse_once(server, match_first, sse1).await;
+    mount_sse_once_match(server, match_first, sse1).await;
 
     let match_compact = |req: &wiremock::Request| {
         let body = std::str::from_utf8(&req.body).unwrap_or("");
         body.contains("You have exceeded the maximum number of tokens")
     };
-    mount_sse_once(server, match_compact, sse2).await;
+    mount_sse_once_match(server, match_compact, sse2).await;
 
     let match_after_compact = |req: &wiremock::Request| {
         let body = std::str::from_utf8(&req.body).unwrap_or("");
@@ -716,19 +716,19 @@ async fn mount_initial_flow(server: &MockServer) {
             && !body.contains("\"text\":\"AFTER_RESUME\"")
             && !body.contains("\"text\":\"AFTER_FORK\"")
     };
-    mount_sse_once(server, match_after_compact, sse3).await;
+    mount_sse_once_match(server, match_after_compact, sse3).await;
 
     let match_after_resume = |req: &wiremock::Request| {
         let body = std::str::from_utf8(&req.body).unwrap_or("");
         body.contains("\"text\":\"AFTER_RESUME\"")
     };
-    mount_sse_once(server, match_after_resume, sse4).await;
+    mount_sse_once_match(server, match_after_resume, sse4).await;
 
     let match_after_fork = |req: &wiremock::Request| {
         let body = std::str::from_utf8(&req.body).unwrap_or("");
         body.contains("\"text\":\"AFTER_FORK\"")
     };
-    mount_sse_once(server, match_after_fork, sse5).await;
+    mount_sse_once_match(server, match_after_fork, sse5).await;
 }
 
 async fn mount_second_compact_flow(server: &MockServer) {
@@ -743,13 +743,13 @@ async fn mount_second_compact_flow(server: &MockServer) {
         body.contains("You have exceeded the maximum number of tokens")
             && body.contains("AFTER_FORK")
     };
-    mount_sse_once(server, match_second_compact, sse6).await;
+    mount_sse_once_match(server, match_second_compact, sse6).await;
 
     let match_after_second_resume = |req: &wiremock::Request| {
         let body = std::str::from_utf8(&req.body).unwrap_or("");
         body.contains(&format!("\"text\":\"{AFTER_SECOND_RESUME}\""))
     };
-    mount_sse_once(server, match_after_second_resume, sse7).await;
+    mount_sse_once_match(server, match_after_second_resume, sse7).await;
 }
 
 async fn start_test_conversation(

codex-rs/core/tests/suite/json_result.rs

@@ -67,7 +67,7 @@ async fn codex_returns_json_result(model: String) -> anyhow::Result<()> {
             && format.get("strict") == Some(&serde_json::Value::Bool(true))
             && format.get("schema") == Some(&expected_schema)
     };
-    responses::mount_sse_once(&server, match_json_text_param, sse1).await;
+    responses::mount_sse_once_match(&server, match_json_text_param, sse1).await;
 
     let TestCodex { codex, cwd, .. } = test_codex().build(&server).await?;
 

codex-rs/core/tests/suite/mod.rs

@@ -12,6 +12,7 @@ mod fork_conversation;
 mod json_result;
 mod live_cli;
 mod model_overrides;
+mod otel;
 mod prompt_caching;
 mod review;
 mod rmcp_client;

codex-rs/core/tests/suite/rmcp_client.rs

@@ -12,7 +12,7 @@ use codex_core::protocol::Op;
 use codex_core::protocol::SandboxPolicy;
 use codex_protocol::config_types::ReasoningSummary;
 use core_test_support::responses;
-use core_test_support::responses::mount_sse_once;
+use core_test_support::responses::mount_sse_once_match;
 use core_test_support::skip_if_no_network;
 use core_test_support::test_codex::test_codex;
 use core_test_support::wait_for_event;
@@ -36,7 +36,7 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> {
     let server_name = "rmcp";
     let tool_name = format!("{server_name}__echo");
 
-    mount_sse_once(
+    mount_sse_once_match(
         &server,
         any(),
         responses::sse(vec![
@@ -49,7 +49,7 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> {
         ]),
     )
     .await;
-    mount_sse_once(
+    mount_sse_once_match(
         &server,
         any(),
         responses::sse(vec![
@@ -173,7 +173,7 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> {
     let server_name = "rmcp_http";
     let tool_name = format!("{server_name}__echo");
 
-    mount_sse_once(
+    mount_sse_once_match(
         &server,
         any(),
         responses::sse(vec![
@@ -186,7 +186,7 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> {
         ]),
     )
     .await;
-    mount_sse_once(
+    mount_sse_once_match(
         &server,
         any(),
         responses::sse(vec![

codex-rs/core/tests/suite/user_notification.rs

@@ -28,7 +28,7 @@ async fn summarize_context_three_requests_and_instructions() -> anyhow::Result<(
 
     let sse1 = sse(vec![ev_assistant_message("m1", "Done"), ev_completed("r1")]);
 
-    responses::mount_sse_once(&server, any(), sse1).await;
+    responses::mount_sse_once_match(&server, any(), sse1).await;
 
     let notify_dir = TempDir::new()?;
     // write a script to the notify that touches a file next to it