Files
llmx/codex-cli/tests/approvals.test.ts

127 lines
4.1 KiB
TypeScript
Raw Normal View History

import type { SafetyAssessment } from "../src/approvals";
import { canAutoApprove } from "../src/approvals";
import { describe, test, expect } from "vitest";
describe("canAutoApprove()", () => {
const env = {
PATH: "/usr/local/bin:/usr/bin:/bin",
HOME: "/home/user",
};
const writeablePaths: Array<string> = [];
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
const check = (
command: ReadonlyArray<string>,
policy: "suggest" | "auto-edit" | "full-auto" = "suggest",
): SafetyAssessment => canAutoApprove(command, policy, writeablePaths, env);
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
test("simple commands in suggest mode should require approval", () => {
// In suggest mode, all commands should require approval
expect(check(["ls"])).toEqual({ type: "ask-user" });
expect(check(["cat", "file.txt"])).toEqual({ type: "ask-user" });
expect(check(["pwd"])).toEqual({ type: "ask-user" });
});
test("simple safe commands in auto-edit mode", () => {
// In auto-edit mode, safe commands should be auto-approved
expect(check(["ls"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "List directory",
group: "Searching",
runInSandbox: false,
});
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
expect(check(["cat", "file.txt"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "View file contents",
group: "Reading files",
runInSandbox: false,
});
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
expect(check(["pwd"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "Print working directory",
group: "Navigating",
runInSandbox: false,
});
});
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
test("bash commands in suggest mode should require approval", () => {
// In suggest mode, all bash commands should require approval
expect(check(["bash", "-lc", "ls"])).toEqual({ type: "ask-user" });
expect(check(["bash", "-lc", "ls $HOME"])).toEqual({ type: "ask-user" });
expect(check(["bash", "-lc", "git show ab9811cb90"])).toEqual({
type: "ask-user",
});
});
test("bash commands in auto-edit mode", () => {
// In auto-edit mode, safe bash commands should be auto-approved
expect(check(["bash", "-lc", "ls"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "List directory",
group: "Searching",
runInSandbox: false,
});
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
expect(check(["bash", "-lc", "ls $HOME"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "List directory",
group: "Searching",
runInSandbox: false,
});
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
expect(check(["bash", "-lc", "git show ab9811cb90"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "Git show",
group: "Using git",
runInSandbox: false,
});
});
test("bash -lc commands with unsafe redirects", () => {
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
// In suggest mode, all commands should require approval
expect(check(["bash", "-lc", "echo hello > file.txt"])).toEqual({
type: "ask-user",
});
expect(check(["bash", "-lc", "ls && pwd"])).toEqual({
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
type: "ask-user",
});
// In auto-edit mode, commands with redirects should still require approval
expect(
check(["bash", "-lc", "echo hello > file.txt"], "auto-edit"),
).toEqual({
type: "ask-user",
});
// In auto-edit mode, safe commands with safe operators should be auto-approved
expect(check(["bash", "-lc", "ls && pwd"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "List directory",
group: "Searching",
runInSandbox: false,
});
});
fix(security): Shell commands auto-executing in 'suggest' mode without permission (#197) ## Problem There's a security vulnerability in the current implementation where shell commands are being executed without requesting user permission even when in 'suggest' mode. According to our documentation: > In **Suggest** mode (default): All file writes/patches and **ALL shell/Bash commands** should require approval. However, the current implementation in `approvals.ts` was auto-approving commands deemed "safe" by the `isSafeCommand` function, bypassing the user permission requirement. This is a security risk as users expect all shell commands to require explicit approval in 'suggest' mode. ## Solution This PR fixes the issue by modifying the `canAutoApprove` function in `approvals.ts` to respect the 'suggest' mode policy for all shell commands: 1. Added an early check at the beginning of `canAutoApprove` to immediately return `{ type: "ask-user" }` when the policy is `suggest`, regardless of whether the command is considered "safe" or not. 2. Added a similar check in the bash command handling section to ensure bash commands also respect the 'suggest' mode. 3. Updated tests to verify the new behavior, ensuring that all shell commands require approval in 'suggest' mode, while still being auto-approved in 'auto-edit' and 'full-auto' modes when appropriate. ## Testing All tests pass, confirming that the fix works as expected. The updated tests verify that: - All commands (even "safe" ones) require approval in 'suggest' mode - Safe commands are still auto-approved in 'auto-edit' mode - Bash commands with redirects still require approval in all modes This change ensures that the behavior matches what's documented and what users expect, improving security by requiring explicit permission for all shell commands in the default 'suggest' mode.
2025-04-18 00:15:02 +10:00
test("true command in suggest mode requires approval", () => {
expect(check(["true"])).toEqual({ type: "ask-user" });
});
test("true command in auto-edit mode is auto-approved", () => {
expect(check(["true"], "auto-edit")).toEqual({
type: "auto-approve",
reason: "Noop (true)",
group: "Utility",
runInSandbox: false,
});
});
test("commands that should require approval", () => {
// Should this be on the auto-approved list?
expect(check(["printenv"])).toEqual({ type: "ask-user" });
expect(check(["git", "commit"])).toEqual({ type: "ask-user" });
expect(check(["pytest"])).toEqual({ type: "ask-user" });
expect(check(["cargo", "build"])).toEqual({ type: "ask-user" });
});
});