add codex debug seatbelt --log-denials (#4098)

This adds a debugging tool for analyzing why certain commands fail to
execute under the sandbox.

Example output:

```
$ codex debug seatbelt --log-denials bash -lc "(echo foo > ~/foo.txt)"
bash: /Users/nornagon/foo.txt: Operation not permitted

=== Sandbox denials ===
(bash) file-write-data /dev/tty
(bash) file-write-data /dev/ttys001
(bash) sysctl-read kern.ngroups
(bash) file-write-create /Users/nornagon/foo.txt
```

It operates by:

1. spawning `log stream` to watch system logs, and
2. tracking all descendant PIDs using kqueue + proc_listchildpids.

this is a "best-effort" technique, as `log stream` may drop logs(?), and
kqueue + proc_listchildpids isn't atomic and can end up missing very
short-lived processes. But it works well enough in my testing to be
useful :)

codex-rs/Cargo.lock

@@ -986,14 +986,17 @@ dependencies = [
  "codex-tui",
  "codex-windows-sandbox",
  "ctor 0.5.0",
+ "libc",
  "owo-colors",
  "predicates",
  "pretty_assertions",
+ "regex-lite",
  "serde_json",
  "supports-color",
  "tempfile",
  "tokio",
  "toml",
+ "tracing",
 ]
 
 [[package]]

codex-rs/README.md

@@ -58,7 +58,7 @@ To test to see what happens when a command is run under the sandbox provided by
 
 ```
 # macOS
-codex sandbox macos [--full-auto] [COMMAND]...
+codex sandbox macos [--full-auto] [--log-denials] [COMMAND]...
 
 # Linux
 codex sandbox linux [--full-auto] [COMMAND]...
@@ -67,7 +67,7 @@ codex sandbox linux [--full-auto] [COMMAND]...
 codex sandbox windows [--full-auto] [COMMAND]...
 
 # Legacy aliases
-codex debug seatbelt [--full-auto] [COMMAND]...
+codex debug seatbelt [--full-auto] [--log-denials] [COMMAND]...
 codex debug landlock [--full-auto] [COMMAND]...
 ```
 

codex-rs/cli/Cargo.toml

@@ -35,7 +35,9 @@ codex-rmcp-client = { workspace = true }
 codex-stdio-to-uds = { workspace = true }
 codex-tui = { workspace = true }
 ctor = { workspace = true }
+libc = { workspace = true }
 owo-colors = { workspace = true }
+regex-lite = { workspace = true}
 serde_json = { workspace = true }
 supports-color = { workspace = true }
 toml = { workspace = true }
@@ -46,6 +48,7 @@ tokio = { workspace = true, features = [
     "rt-multi-thread",
     "signal",
 ] }
+tracing = { workspace = true }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 codex_windows_sandbox = { package = "codex-windows-sandbox", path = "../windows-sandbox-rs" }

codex-rs/cli/src/debug_sandbox.rs

@@ -1,3 +1,8 @@
+#[cfg(target_os = "macos")]
+mod pid_tracker;
+#[cfg(target_os = "macos")]
+mod seatbelt;
+
 use std::path::PathBuf;
 
 use codex_common::CliConfigOverrides;
@@ -15,6 +20,9 @@ use crate::SeatbeltCommand;
 use crate::WindowsCommand;
 use crate::exit_status::handle_exit_status;
 
+#[cfg(target_os = "macos")]
+use seatbelt::DenialLogger;
+
 #[cfg(target_os = "macos")]
 pub async fn run_command_under_seatbelt(
     command: SeatbeltCommand,
@@ -22,6 +30,7 @@ pub async fn run_command_under_seatbelt(
 ) -> anyhow::Result<()> {
     let SeatbeltCommand {
         full_auto,
+        log_denials,
         config_overrides,
         command,
     } = command;
@@ -31,6 +40,7 @@ pub async fn run_command_under_seatbelt(
         config_overrides,
         codex_linux_sandbox_exe,
         SandboxType::Seatbelt,
+        log_denials,
     )
     .await
 }
@@ -58,6 +68,7 @@ pub async fn run_command_under_landlock(
         config_overrides,
         codex_linux_sandbox_exe,
         SandboxType::Landlock,
+        false,
     )
     .await
 }
@@ -77,6 +88,7 @@ pub async fn run_command_under_windows(
         config_overrides,
         codex_linux_sandbox_exe,
         SandboxType::Windows,
+        false,
     )
     .await
 }
@@ -94,6 +106,7 @@ async fn run_command_under_sandbox(
     config_overrides: CliConfigOverrides,
     codex_linux_sandbox_exe: Option<PathBuf>,
     sandbox_type: SandboxType,
+    log_denials: bool,
 ) -> anyhow::Result<()> {
     let sandbox_mode = create_sandbox_mode(full_auto);
     let config = Config::load_with_cli_overrides(
@@ -180,6 +193,11 @@ async fn run_command_under_sandbox(
         }
     }
 
+    #[cfg(target_os = "macos")]
+    let mut denial_logger = log_denials.then(DenialLogger::new).flatten();
+    #[cfg(not(target_os = "macos"))]
+    let _ = log_denials;
+
     let mut child = match sandbox_type {
         #[cfg(target_os = "macos")]
         SandboxType::Seatbelt => {
@@ -213,8 +231,27 @@ async fn run_command_under_sandbox(
             unreachable!("Windows sandbox should have been handled above");
         }
     };
+
+    #[cfg(target_os = "macos")]
+    if let Some(denial_logger) = &mut denial_logger {
+        denial_logger.on_child_spawn(&child);
+    }
+
     let status = child.wait().await?;
 
+    #[cfg(target_os = "macos")]
+    if let Some(denial_logger) = denial_logger {
+        let denials = denial_logger.finish().await;
+        eprintln!("\n=== Sandbox denials ===");
+        if denials.is_empty() {
+            eprintln!("None found.");
+        } else {
+            for seatbelt::SandboxDenial { name, capability } in denials {
+                eprintln!("({name}) {capability}");
+            }
+        }
+    }
+
     handle_exit_status(status);
 }
 

codex-rs/cli/src/debug_sandbox/pid_tracker.rs

@@ -0,0 +1,372 @@
+use std::collections::HashSet;
+use tokio::task::JoinHandle;
+use tracing::warn;
+
+/// Tracks the (recursive) descendants of a process by using `kqueue` to watch for fork events, and
+/// `proc_listchildpids` to list the children of a process.
+pub(crate) struct PidTracker {
+    kq: libc::c_int,
+    handle: JoinHandle<HashSet<i32>>,
+}
+
+impl PidTracker {
+    pub(crate) fn new(root_pid: i32) -> Option<Self> {
+        if root_pid <= 0 {
+            return None;
+        }
+
+        let kq = unsafe { libc::kqueue() };
+        let handle = tokio::task::spawn_blocking(move || track_descendants(kq, root_pid));
+
+        Some(Self { kq, handle })
+    }
+
+    pub(crate) async fn stop(self) -> HashSet<i32> {
+        trigger_stop_event(self.kq);
+        self.handle.await.unwrap_or_default()
+    }
+}
+
+unsafe extern "C" {
+    fn proc_listchildpids(
+        ppid: libc::c_int,
+        buffer: *mut libc::c_void,
+        buffersize: libc::c_int,
+    ) -> libc::c_int;
+}
+
+/// Wrap proc_listchildpids.
+fn list_child_pids(parent: i32) -> Vec<i32> {
+    unsafe {
+        let mut capacity: usize = 16;
+        loop {
+            let mut buf: Vec<i32> = vec![0; capacity];
+            let count = proc_listchildpids(
+                parent as libc::c_int,
+                buf.as_mut_ptr() as *mut libc::c_void,
+                (buf.len() * std::mem::size_of::<i32>()) as libc::c_int,
+            );
+            if count <= 0 {
+                return Vec::new();
+            }
+            let returned = count as usize;
+            if returned < capacity {
+                buf.truncate(returned);
+                return buf;
+            }
+            capacity = capacity.saturating_mul(2).max(returned + 16);
+        }
+    }
+}
+
+fn pid_is_alive(pid: i32) -> bool {
+    if pid <= 0 {
+        return false;
+    }
+    let res = unsafe { libc::kill(pid as libc::pid_t, 0) };
+    if res == 0 {
+        true
+    } else {
+        matches!(
+            std::io::Error::last_os_error().raw_os_error(),
+            Some(libc::EPERM)
+        )
+    }
+}
+
+enum WatchPidError {
+    ProcessGone,
+    Other(std::io::Error),
+}
+
+/// Add `pid` to the watch list in `kq`.
+fn watch_pid(kq: libc::c_int, pid: i32) -> Result<(), WatchPidError> {
+    if pid <= 0 {
+        return Err(WatchPidError::ProcessGone);
+    }
+
+    let kev = libc::kevent {
+        ident: pid as libc::uintptr_t,
+        filter: libc::EVFILT_PROC,
+        flags: libc::EV_ADD | libc::EV_CLEAR,
+        fflags: libc::NOTE_FORK | libc::NOTE_EXEC | libc::NOTE_EXIT,
+        data: 0,
+        udata: std::ptr::null_mut(),
+    };
+
+    let res = unsafe { libc::kevent(kq, &kev, 1, std::ptr::null_mut(), 0, std::ptr::null()) };
+    if res < 0 {
+        let err = std::io::Error::last_os_error();
+        if err.raw_os_error() == Some(libc::ESRCH) {
+            Err(WatchPidError::ProcessGone)
+        } else {
+            Err(WatchPidError::Other(err))
+        }
+    } else {
+        Ok(())
+    }
+}
+
+fn watch_children(
+    kq: libc::c_int,
+    parent: i32,
+    seen: &mut HashSet<i32>,
+    active: &mut HashSet<i32>,
+) {
+    for child_pid in list_child_pids(parent) {
+        add_pid_watch(kq, child_pid, seen, active);
+    }
+}
+
+/// Watch `pid` and its children, updating `seen` and `active` sets.
+fn add_pid_watch(kq: libc::c_int, pid: i32, seen: &mut HashSet<i32>, active: &mut HashSet<i32>) {
+    if pid <= 0 {
+        return;
+    }
+
+    let newly_seen = seen.insert(pid);
+    let mut should_recurse = newly_seen;
+
+    if active.insert(pid) {
+        match watch_pid(kq, pid) {
+            Ok(()) => {
+                should_recurse = true;
+            }
+            Err(WatchPidError::ProcessGone) => {
+                active.remove(&pid);
+                return;
+            }
+            Err(WatchPidError::Other(err)) => {
+                warn!("failed to watch pid {pid}: {err}");
+                active.remove(&pid);
+                return;
+            }
+        }
+    }
+
+    if should_recurse {
+        watch_children(kq, pid, seen, active);
+    }
+}
+const STOP_IDENT: libc::uintptr_t = 1;
+
+fn register_stop_event(kq: libc::c_int) -> bool {
+    let kev = libc::kevent {
+        ident: STOP_IDENT,
+        filter: libc::EVFILT_USER,
+        flags: libc::EV_ADD | libc::EV_CLEAR,
+        fflags: 0,
+        data: 0,
+        udata: std::ptr::null_mut(),
+    };
+
+    let res = unsafe { libc::kevent(kq, &kev, 1, std::ptr::null_mut(), 0, std::ptr::null()) };
+    res >= 0
+}
+
+fn trigger_stop_event(kq: libc::c_int) {
+    if kq < 0 {
+        return;
+    }
+
+    let kev = libc::kevent {
+        ident: STOP_IDENT,
+        filter: libc::EVFILT_USER,
+        flags: 0,
+        fflags: libc::NOTE_TRIGGER,
+        data: 0,
+        udata: std::ptr::null_mut(),
+    };
+
+    let _ = unsafe { libc::kevent(kq, &kev, 1, std::ptr::null_mut(), 0, std::ptr::null()) };
+}
+
+/// Put all of the above together to track all the descendants of `root_pid`.
+fn track_descendants(kq: libc::c_int, root_pid: i32) -> HashSet<i32> {
+    if kq < 0 {
+        let mut seen = HashSet::new();
+        seen.insert(root_pid);
+        return seen;
+    }
+
+    if !register_stop_event(kq) {
+        let mut seen = HashSet::new();
+        seen.insert(root_pid);
+        let _ = unsafe { libc::close(kq) };
+        return seen;
+    }
+
+    let mut seen: HashSet<i32> = HashSet::new();
+    let mut active: HashSet<i32> = HashSet::new();
+
+    add_pid_watch(kq, root_pid, &mut seen, &mut active);
+
+    const EVENTS_CAP: usize = 32;
+    let mut events: [libc::kevent; EVENTS_CAP] =
+        unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
+
+    let mut stop_requested = false;
+    loop {
+        if active.is_empty() {
+            if !pid_is_alive(root_pid) {
+                break;
+            }
+            add_pid_watch(kq, root_pid, &mut seen, &mut active);
+            if active.is_empty() {
+                continue;
+            }
+        }
+
+        let nev = unsafe {
+            libc::kevent(
+                kq,
+                std::ptr::null::<libc::kevent>(),
+                0,
+                events.as_mut_ptr(),
+                EVENTS_CAP as libc::c_int,
+                std::ptr::null(),
+            )
+        };
+
+        if nev < 0 {
+            let err = std::io::Error::last_os_error();
+            if err.kind() == std::io::ErrorKind::Interrupted {
+                continue;
+            }
+            break;
+        }
+
+        if nev == 0 {
+            continue;
+        }
+
+        for ev in events.iter().take(nev as usize) {
+            let pid = ev.ident as i32;
+
+            if ev.filter == libc::EVFILT_USER && ev.ident == STOP_IDENT {
+                stop_requested = true;
+                break;
+            }
+
+            if (ev.flags & libc::EV_ERROR) != 0 {
+                if ev.data == libc::ESRCH as isize {
+                    active.remove(&pid);
+                }
+                continue;
+            }
+
+            if (ev.fflags & libc::NOTE_FORK) != 0 {
+                watch_children(kq, pid, &mut seen, &mut active);
+            }
+
+            if (ev.fflags & libc::NOTE_EXIT) != 0 {
+                active.remove(&pid);
+            }
+        }
+
+        if stop_requested {
+            break;
+        }
+    }
+
+    let _ = unsafe { libc::close(kq) };
+
+    seen
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::process::Command;
+    use std::process::Stdio;
+    use std::time::Duration;
+
+    #[test]
+    fn pid_is_alive_detects_current_process() {
+        let pid = std::process::id() as i32;
+        assert!(pid_is_alive(pid));
+    }
+
+    #[cfg(target_os = "macos")]
+    #[test]
+    fn list_child_pids_includes_spawned_child() {
+        let mut child = Command::new("/bin/sleep")
+            .arg("5")
+            .stdin(Stdio::null())
+            .spawn()
+            .expect("failed to spawn child process");
+
+        let child_pid = child.id() as i32;
+        let parent_pid = std::process::id() as i32;
+
+        let mut found = false;
+        for _ in 0..100 {
+            if list_child_pids(parent_pid).contains(&child_pid) {
+                found = true;
+                break;
+            }
+            std::thread::sleep(Duration::from_millis(10));
+        }
+
+        let _ = child.kill();
+        let _ = child.wait();
+
+        assert!(found, "expected to find child pid {child_pid} in list");
+    }
+
+    #[cfg(target_os = "macos")]
+    #[tokio::test]
+    async fn pid_tracker_collects_spawned_children() {
+        let tracker = PidTracker::new(std::process::id() as i32).expect("failed to create tracker");
+
+        let mut child = Command::new("/bin/sleep")
+            .arg("0.1")
+            .stdin(Stdio::null())
+            .spawn()
+            .expect("failed to spawn child process");
+
+        let child_pid = child.id() as i32;
+        let parent_pid = std::process::id() as i32;
+
+        let _ = child.wait();
+
+        let seen = tracker.stop().await;
+
+        assert!(
+            seen.contains(&parent_pid),
+            "expected tracker to include parent pid {parent_pid}"
+        );
+        assert!(
+            seen.contains(&child_pid),
+            "expected tracker to include child pid {child_pid}"
+        );
+    }
+
+    #[cfg(target_os = "macos")]
+    #[tokio::test]
+    async fn pid_tracker_collects_bash_subshell_descendants() {
+        let tracker = PidTracker::new(std::process::id() as i32).expect("failed to create tracker");
+
+        let child = Command::new("/bin/bash")
+            .arg("-c")
+            .arg("(sleep 0.1 & echo $!; wait)")
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::null())
+            .spawn()
+            .expect("failed to spawn bash");
+
+        let output = child.wait_with_output().unwrap().stdout;
+        let subshell_pid = String::from_utf8_lossy(&output)
+            .trim()
+            .parse::<i32>()
+            .expect("failed to parse subshell pid");
+
+        let seen = tracker.stop().await;
+
+        assert!(
+            seen.contains(&subshell_pid),
+            "expected tracker to include subshell pid {subshell_pid}"
+        );
+    }
+}

codex-rs/cli/src/debug_sandbox/seatbelt.rs

@@ -0,0 +1,114 @@
+use std::collections::HashSet;
+use tokio::io::AsyncBufReadExt;
+use tokio::process::Child;
+use tokio::task::JoinHandle;
+
+use super::pid_tracker::PidTracker;
+
+pub struct SandboxDenial {
+    pub name: String,
+    pub capability: String,
+}
+
+pub struct DenialLogger {
+    log_stream: Child,
+    pid_tracker: Option<PidTracker>,
+    log_reader: Option<JoinHandle<Vec<u8>>>,
+}
+
+impl DenialLogger {
+    pub(crate) fn new() -> Option<Self> {
+        let mut log_stream = start_log_stream()?;
+        let stdout = log_stream.stdout.take()?;
+        let log_reader = tokio::spawn(async move {
+            let mut reader = tokio::io::BufReader::new(stdout);
+            let mut logs = Vec::new();
+            let mut chunk = Vec::new();
+            loop {
+                match reader.read_until(b'\n', &mut chunk).await {
+                    Ok(0) | Err(_) => break,
+                    Ok(_) => {
+                        logs.extend_from_slice(&chunk);
+                        chunk.clear();
+                    }
+                }
+            }
+            logs
+        });
+
+        Some(Self {
+            log_stream,
+            pid_tracker: None,
+            log_reader: Some(log_reader),
+        })
+    }
+
+    pub(crate) fn on_child_spawn(&mut self, child: &Child) {
+        if let Some(root_pid) = child.id() {
+            self.pid_tracker = PidTracker::new(root_pid as i32);
+        }
+    }
+
+    pub(crate) async fn finish(mut self) -> Vec<SandboxDenial> {
+        let pid_set = match self.pid_tracker {
+            Some(tracker) => tracker.stop().await,
+            None => Default::default(),
+        };
+
+        if pid_set.is_empty() {
+            return Vec::new();
+        }
+
+        let _ = self.log_stream.kill().await;
+        let _ = self.log_stream.wait().await;
+
+        let logs_bytes = match self.log_reader.take() {
+            Some(handle) => handle.await.unwrap_or_default(),
+            None => Vec::new(),
+        };
+        let logs = String::from_utf8_lossy(&logs_bytes);
+
+        let mut seen: HashSet<(String, String)> = HashSet::new();
+        let mut denials: Vec<SandboxDenial> = Vec::new();
+        for line in logs.lines() {
+            if let Ok(json) = serde_json::from_str::<serde_json::Value>(line)
+                && let Some(msg) = json.get("eventMessage").and_then(|v| v.as_str())
+                && let Some((pid, name, capability)) = parse_message(msg)
+                && pid_set.contains(&pid)
+                && seen.insert((name.clone(), capability.clone()))
+            {
+                denials.push(SandboxDenial { name, capability });
+            }
+        }
+        denials
+    }
+}
+
+fn start_log_stream() -> Option<Child> {
+    use std::process::Stdio;
+
+    const PREDICATE: &str = r#"(((processID == 0) AND (senderImagePath CONTAINS "/Sandbox")) OR (subsystem == "com.apple.sandbox.reporting"))"#;
+
+    tokio::process::Command::new("log")
+        .args(["stream", "--style", "ndjson", "--predicate", PREDICATE])
+        .stdin(Stdio::null())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .kill_on_drop(true)
+        .spawn()
+        .ok()
+}
+
+fn parse_message(msg: &str) -> Option<(i32, String, String)> {
+    // Example message:
+    // Sandbox: processname(1234) deny(1) capability-name args...
+    static RE: std::sync::OnceLock<regex_lite::Regex> = std::sync::OnceLock::new();
+    let re = RE.get_or_init(|| {
+        #[expect(clippy::unwrap_used)]
+        regex_lite::Regex::new(r"^Sandbox:\s*(.+?)\((\d+)\)\s+deny\(.*?\)\s*(.+)$").unwrap()
+    });
+
+    let (_, [name, pid_str, capability]) = re.captures(msg)?.extract();
+    let pid = pid_str.trim().parse::<i32>().ok()?;
+    Some((pid, name.to_string(), capability.to_string()))
+}

codex-rs/cli/src/lib.rs

@@ -11,6 +11,10 @@ pub struct SeatbeltCommand {
     #[arg(long = "full-auto", default_value_t = false)]
     pub full_auto: bool,
 
+    /// While the command runs, capture macOS sandbox denials via `log stream` and print them after exit
+    #[arg(long = "log-denials", default_value_t = false)]
+    pub log_denials: bool,
+
     #[clap(skip)]
     pub config_overrides: CliConfigOverrides,