fix: separate codex mcp into codex mcp-server and codex app-server (#4471)

This is a very large PR with some non-backwards-compatible changes.

Historically, `codex mcp` (or `codex mcp serve`) started a JSON-RPC-ish
server that had two overlapping responsibilities:

- Running an MCP server, providing some basic tool calls.
- Running the app server used to power experiences such as the VS Code
extension.

This PR aims to separate these into distinct concepts:

- `codex mcp-server` for the MCP server
- `codex app-server` for the "application server"

Note `codex mcp` still exists because it already has its own subcommands
for MCP management (`list`, `add`, etc.)

The MCP logic continues to live in `codex-rs/mcp-server` whereas the
refactored app server logic is in the new `codex-rs/app-server` folder.
Note that most of the existing integration tests in
`codex-rs/mcp-server/tests/suite` were actually for the app server, so
all the tests have been moved with the exception of
`codex-rs/mcp-server/tests/suite/mod.rs`.

Because this is already a large diff, I tried not to change more than I
had to, so `codex-rs/app-server/tests/common/mcp_process.rs` still uses
the name `McpProcess` for now, but I will do some mechanical renamings
to things like `AppServer` in subsequent PRs.

While `mcp-server` and `app-server` share some overlapping functionality
(like reading streams of JSONL and dispatching based on message types)
and some differences (completely different message types), I ended up
doing a bit of copypasta between the two crates, as both have somewhat
similar `message_processor.rs` and `outgoing_message.rs` files for now,
though I expect them to diverge more in the near future.

One material change is that of the initialize handshake for `codex
app-server`, as we no longer use the MCP types for that handshake.
Instead, we update `codex-rs/protocol/src/mcp_protocol.rs` to add an
`Initialize` variant to `ClientRequest`, which takes the `ClientInfo`
object we need to update the `USER_AGENT_SUFFIX` in
`codex-rs/app-server/src/message_processor.rs`.

One other material change is in
`codex-rs/app-server/src/codex_message_processor.rs` where I eliminated
a use of the `send_event_as_notification()` method I am generally trying
to deprecate (because it blindly maps an `EventMsg` into a
`JSONNotification`) in favor of `send_server_notification()`, which
takes a `ServerNotification`, as that is intended to be a custom enum of
all notification types supported by the app server. So to make this
update, I had to introduce a new variant of `ServerNotification`,
`SessionConfigured`, which is a non-backwards compatible change with the
old `codex mcp`, and clients will have to be updated after the next
release that contains this PR. Note that
`codex-rs/app-server/tests/suite/list_resume.rs` also had to be update
to reflect this change.

I introduced `codex-rs/utils/json-to-toml/src/lib.rs` as a small utility
crate to avoid some of the copying between `mcp-server` and
`app-server`.

codex-rs/mcp-server/src/outgoing_message.rs

@@ -3,7 +3,6 @@ use std::sync::atomic::AtomicI64;
 use std::sync::atomic::Ordering;
 
 use codex_core::protocol::Event;
-use codex_protocol::mcp_protocol::ServerNotification;
 use mcp_types::JSONRPC_VERSION;
 use mcp_types::JSONRPCError;
 use mcp_types::JSONRPCErrorError;
@@ -125,12 +124,6 @@ impl OutgoingMessageSender {
         .await;
     }
 
-    pub(crate) async fn send_server_notification(&self, notification: ServerNotification) {
-        let _ = self
-            .sender
-            .send(OutgoingMessage::AppServerNotification(notification));
-    }
-
     pub(crate) async fn send_notification(&self, notification: OutgoingNotification) {
         let outgoing_message = OutgoingMessage::Notification(notification);
         let _ = self.sender.send(outgoing_message);
@@ -146,9 +139,6 @@ impl OutgoingMessageSender {
 pub(crate) enum OutgoingMessage {
     Request(OutgoingRequest),
     Notification(OutgoingNotification),
-    /// AppServerNotification is specific to the case where this is run as an
-    /// "app server" as opposed to an MCP server.
-    AppServerNotification(ServerNotification),
     Response(OutgoingResponse),
     Error(OutgoingError),
 }
@@ -172,21 +162,6 @@ impl From<OutgoingMessage> for JSONRPCMessage {
                     params,
                 })
             }
-            AppServerNotification(notification) => {
-                let method = notification.to_string();
-                let params = match notification.to_params() {
-                    Ok(params) => Some(params),
-                    Err(err) => {
-                        warn!("failed to serialize notification params: {err}");
-                        None
-                    }
-                };
-                JSONRPCMessage::Notification(JSONRPCNotification {
-                    jsonrpc: JSONRPC_VERSION.into(),
-                    method,
-                    params,
-                })
-            }
             Response(OutgoingResponse { id, result }) => {
                 JSONRPCMessage::Response(JSONRPCResponse {
                     jsonrpc: JSONRPC_VERSION.into(),
@@ -261,11 +236,9 @@ mod tests {
     use codex_core::protocol::SessionConfiguredEvent;
     use codex_protocol::config_types::ReasoningEffort;
     use codex_protocol::mcp_protocol::ConversationId;
-    use codex_protocol::mcp_protocol::LoginChatGptCompleteNotification;
     use pretty_assertions::assert_eq;
     use serde_json::json;
     use tempfile::NamedTempFile;
-    use uuid::Uuid;
 
     use super::*;
 
@@ -357,29 +330,4 @@ mod tests {
         assert_eq!(params.unwrap(), expected_params);
         Ok(())
     }
-
-    #[test]
-    fn verify_server_notification_serialization() {
-        let notification =
-            ServerNotification::LoginChatGptComplete(LoginChatGptCompleteNotification {
-                login_id: Uuid::nil(),
-                success: true,
-                error: None,
-            });
-
-        let jsonrpc_notification: JSONRPCMessage =
-            OutgoingMessage::AppServerNotification(notification).into();
-        assert_eq!(
-            JSONRPCMessage::Notification(JSONRPCNotification {
-                jsonrpc: "2.0".into(),
-                method: "loginChatGptComplete".into(),
-                params: Some(json!({
-                    "loginId": Uuid::nil(),
-                    "success": true,
-                })),
-            }),
-            jsonrpc_notification,
-            "ensure the strum macros serialize the method field correctly"
-        );
-    }
 }