fix: separate codex mcp into codex mcp-server and codex app-server (#4471)

This is a very large PR with some non-backwards-compatible changes.

Historically, `codex mcp` (or `codex mcp serve`) started a JSON-RPC-ish
server that had two overlapping responsibilities:

- Running an MCP server, providing some basic tool calls.
- Running the app server used to power experiences such as the VS Code
extension.

This PR aims to separate these into distinct concepts:

- `codex mcp-server` for the MCP server
- `codex app-server` for the "application server"

Note `codex mcp` still exists because it already has its own subcommands
for MCP management (`list`, `add`, etc.)

The MCP logic continues to live in `codex-rs/mcp-server` whereas the
refactored app server logic is in the new `codex-rs/app-server` folder.
Note that most of the existing integration tests in
`codex-rs/mcp-server/tests/suite` were actually for the app server, so
all the tests have been moved with the exception of
`codex-rs/mcp-server/tests/suite/mod.rs`.

Because this is already a large diff, I tried not to change more than I
had to, so `codex-rs/app-server/tests/common/mcp_process.rs` still uses
the name `McpProcess` for now, but I will do some mechanical renamings
to things like `AppServer` in subsequent PRs.

While `mcp-server` and `app-server` share some overlapping functionality
(like reading streams of JSONL and dispatching based on message types)
and some differences (completely different message types), I ended up
doing a bit of copypasta between the two crates, as both have somewhat
similar `message_processor.rs` and `outgoing_message.rs` files for now,
though I expect them to diverge more in the near future.

One material change is that of the initialize handshake for `codex
app-server`, as we no longer use the MCP types for that handshake.
Instead, we update `codex-rs/protocol/src/mcp_protocol.rs` to add an
`Initialize` variant to `ClientRequest`, which takes the `ClientInfo`
object we need to update the `USER_AGENT_SUFFIX` in
`codex-rs/app-server/src/message_processor.rs`.

One other material change is in
`codex-rs/app-server/src/codex_message_processor.rs` where I eliminated
a use of the `send_event_as_notification()` method I am generally trying
to deprecate (because it blindly maps an `EventMsg` into a
`JSONNotification`) in favor of `send_server_notification()`, which
takes a `ServerNotification`, as that is intended to be a custom enum of
all notification types supported by the app server. So to make this
update, I had to introduce a new variant of `ServerNotification`,
`SessionConfigured`, which is a non-backwards compatible change with the
old `codex mcp`, and clients will have to be updated after the next
release that contains this PR. Note that
`codex-rs/app-server/tests/suite/list_resume.rs` also had to be update
to reflect this change.

I introduced `codex-rs/utils/json-to-toml/src/lib.rs` as a small utility
crate to avoid some of the copying between `mcp-server` and
`app-server`.

codex-rs/protocol/src/mcp_protocol.rs

@@ -12,6 +12,7 @@ use crate::protocol::FileChange;
 use crate::protocol::ReviewDecision;
 use crate::protocol::SandboxPolicy;
 use crate::protocol::TurnAbortReason;
+use mcp_types::JSONRPCNotification;
 use mcp_types::RequestId;
 use serde::Deserialize;
 use serde::Serialize;
@@ -92,6 +93,11 @@ pub enum AuthMode {
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, TS)]
 #[serde(tag = "method", rename_all = "camelCase")]
 pub enum ClientRequest {
+    Initialize {
+        #[serde(rename = "id")]
+        request_id: RequestId,
+        params: InitializeParams,
+    },
     NewConversation {
         #[serde(rename = "id")]
         request_id: RequestId,
@@ -197,6 +203,27 @@ pub enum ClientRequest {
     },
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, TS)]
+#[serde(rename_all = "camelCase")]
+pub struct InitializeParams {
+    pub client_info: ClientInfo,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, TS)]
+#[serde(rename_all = "camelCase")]
+pub struct ClientInfo {
+    pub name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+    pub version: String,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, TS)]
+#[serde(rename_all = "camelCase")]
+pub struct InitializeResponse {
+    pub user_agent: String,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, TS)]
 #[serde(rename_all = "camelCase")]
 pub struct NewConversationParams {
@@ -702,6 +729,33 @@ pub struct LoginChatGptCompleteNotification {
     pub error: Option<String>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, TS)]
+#[serde(rename_all = "camelCase")]
+pub struct SessionConfiguredNotification {
+    /// Name left as session_id instead of conversation_id for backwards compatibility.
+    pub session_id: ConversationId,
+
+    /// Tell the client what model is being queried.
+    pub model: String,
+
+    /// The effort the model is putting into reasoning about the user's request.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub reasoning_effort: Option<ReasoningEffort>,
+
+    /// Identifier of the history log file (inode on Unix, 0 otherwise).
+    pub history_log_id: u64,
+
+    /// Current number of entries in the history log.
+    pub history_entry_count: usize,
+
+    /// Optional initial messages (as events) for resumed sessions.
+    /// When present, UIs can use these to seed the history.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub initial_messages: Option<Vec<EventMsg>>,
+
+    pub rollout_path: PathBuf,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, TS)]
 #[serde(rename_all = "camelCase")]
 pub struct AuthStatusChangeNotification {
@@ -710,7 +764,8 @@ pub struct AuthStatusChangeNotification {
     pub auth_method: Option<AuthMode>,
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, TS, Display)]
+/// Notification sent from the server to the client.
+#[derive(Serialize, Deserialize, Debug, Clone, TS, Display)]
 #[serde(tag = "method", content = "params", rename_all = "camelCase")]
 #[strum(serialize_all = "camelCase")]
 pub enum ServerNotification {
@@ -719,6 +774,9 @@ pub enum ServerNotification {
 
     /// ChatGPT login flow completed
     LoginChatGptComplete(LoginChatGptCompleteNotification),
+
+    /// The special session configured event for a new or resumed conversation.
+    SessionConfigured(SessionConfiguredNotification),
 }
 
 impl ServerNotification {
@@ -726,10 +784,27 @@ impl ServerNotification {
         match self {
             ServerNotification::AuthStatusChange(params) => serde_json::to_value(params),
             ServerNotification::LoginChatGptComplete(params) => serde_json::to_value(params),
+            ServerNotification::SessionConfigured(params) => serde_json::to_value(params),
         }
     }
 }
 
+impl TryFrom<JSONRPCNotification> for ServerNotification {
+    type Error = serde_json::Error;
+
+    fn try_from(value: JSONRPCNotification) -> Result<Self, Self::Error> {
+        serde_json::from_value(serde_json::to_value(value)?)
+    }
+}
+
+/// Notification sent from the client to the server.
+#[derive(Serialize, Deserialize, Debug, Clone, TS, Display)]
+#[serde(tag = "method", content = "params", rename_all = "camelCase")]
+#[strum(serialize_all = "camelCase")]
+pub enum ClientNotification {
+    Initialized,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -795,4 +870,17 @@ mod tests {
         );
         Ok(())
     }
+
+    #[test]
+    fn serialize_client_notification() -> Result<()> {
+        let notification = ClientNotification::Initialized;
+        // Note there is no "params" field for this notification.
+        assert_eq!(
+            json!({
+                "method": "initialized",
+            }),
+            serde_json::to_value(&notification)?,
+        );
+        Ok(())
+    }
 }