Files
llmx/codex-cli/tests/text-buffer.test.ts
moppywhip bc4e6db749 feat: @mention files in codex (#701)
Solves #700

## State of the World Before

Prior to this PR, when users wanted to share file contents with Codex,
they had two options:
- Manually copy and paste file contents into the chat
- Wait for the assistant to use the shell tool to view the file

The second approach required the assistant to:
1. Recognize the need to view a file
2. Execute a shell tool call
3. Wait for the tool call to complete
4. Process the file contents

This consumed extra tokens and reduced user control over which files
were shared with the model.

## State of the World After

With this PR, users can now:
- Reference files directly in their chat input using the `@path` syntax
- Have file contents automatically expanded into XML blocks before being
sent to the LLM

For example, users can type `@src/utils/config.js` in their message, and
the file contents will be included in context. Within the terminal chat
history, these file blocks will be collapsed back to `@path` format in
the UI for clean presentation.

Tag File suggestions:
<img width="857" alt="file-suggestions"
src="https://github.com/user-attachments/assets/397669dc-ad83-492d-b5f0-164fab2ff4ba"
/>

Tagging files in action:
<img width="858" alt="tagging-files"
src="https://github.com/user-attachments/assets/0de9d559-7b7f-4916-aeff-87ae9b16550a"
/>

Demo video of file tagging:
[![Demo video of file
tagging](https://img.youtube.com/vi/vL4LqtBnqt8/0.jpg)](https://www.youtube.com/watch?v=vL4LqtBnqt8)

## Implementation Details

This PR consists of 2 main components:

1. **File Tag Utilities**:
- New `file-tag-utils.ts` utility module that handles both expansion and
collapsing of file tags
- `expandFileTags()` identifies `@path` tokens and replaces them with
XML blocks containing file contents
- `collapseXmlBlocks()` reverses the process, converting XML blocks back
to `@path` format for UI display
- Tokens are only expanded if they point to valid files (directories are
ignored)
   - Expansion happens just before sending input to the model

2. **Terminal Chat Integration**:
- Leveraged the existing file system completion system for tabbing to
support the `@path` syntax
   - Added `updateFsSuggestions` helper to manage filesystem suggestions
- Added `replaceFileSystemSuggestion` to replace input with filesystem
suggestions
- Applied `collapseXmlBlocks` in the chat response rendering so that
tagged files are shown as simple `@path` tags

The PR also includes test coverage for both the UI and the file tag
utilities.

## Next Steps

Some ideas I'd like to implement if this feature gets merged:

- Line selection: `@path[50:80]` to grab specific sections of files
- Method selection: `@path#methodName` to grab just one function/class
- Visual improvements: highlight file tags in the UI to make them more
noticeable
2025-04-30 16:19:55 -07:00

292 lines
10 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import TextBuffer from "../src/text-buffer";
import { describe, it, expect } from "vitest";
describe("TextBuffer basic editing parity with Rust suite", () => {
/* ------------------------------------------------------------------ */
/* insert_char */
/* ------------------------------------------------------------------ */
it("insert_char / printable (single line)", () => {
// (col, char, expectedLine)
const cases: Array<[number, string, string]> = [
[0, "x", "xab"],
[1, "x", "axb"],
[2, "x", "abx"],
[1, "あ", "aあb"],
];
for (const [col, ch, want] of cases) {
const buf = new TextBuffer("ab");
buf.move("end"); // go to col 2
while (buf.getCursor()[1] > col) {
buf.move("left");
}
buf.insert(ch);
expect(buf.getText()).toBe(want);
expect(buf.getCursor()).toEqual([0, col + 1]);
}
});
/* ------------------------------------------------------------------ */
/* insert_char newline support */
/* ------------------------------------------------------------------ */
it("insert_char with a newline should split the line", () => {
const buf = new TextBuffer("ab");
// jump to end of first (and only) line
buf.move("end");
// Insert a raw \n character the Rust implementation splits the line
buf.insert("\n");
// We expect the text to be split into two separate lines
expect(buf.getLines()).toEqual(["ab", ""]);
expect(buf.getCursor()).toEqual([1, 0]);
});
/* ------------------------------------------------------------------ */
/* insert_str helpers */
/* ------------------------------------------------------------------ */
it("insert_str should insert multiline strings", () => {
const initial = ["ab", "cd", "ef"].join("\n");
const buf = new TextBuffer(initial);
// place cursor at (row:0, col:0)
// No move needed cursor starts at 0,0
buf.insertStr("x\ny");
const wantLines = ["x", "yab", "cd", "ef"];
expect(buf.getLines()).toEqual(wantLines);
expect(buf.getCursor()).toEqual([1, 1]);
});
/* ------------------------------------------------------------------ */
/* Undo / Redo */
/* ------------------------------------------------------------------ */
it("undo / redo history should revert edits", () => {
const buf = new TextBuffer("hello");
buf.move("end");
buf.insert("!"); // text becomes "hello!"
expect(buf.undo()).toBe(true);
expect(buf.getText()).toBe("hello");
expect(buf.redo()).toBe(true);
expect(buf.getText()).toBe("hello!");
});
/* ------------------------------------------------------------------ */
/* Selection model */
/* ------------------------------------------------------------------ */
it("copy & paste should operate on current selection", () => {
const buf = new TextBuffer("hello world");
buf.startSelection();
// Select the word "hello"
buf.move("right"); // h
buf.move("right"); // e
buf.move("right"); // l
buf.move("right"); // l
buf.move("right"); // o
buf.endSelection();
buf.copy();
// Move to end and paste
buf.move("end");
// add one space before pasting copied word
buf.insert(" ");
buf.paste();
expect(buf.getText()).toBe("hello world hello");
});
/* ------------------------------------------------------------------ */
/* Backspace behaviour */
/* ------------------------------------------------------------------ */
describe("backspace", () => {
it("deletes the character to the *left* of the caret within a line", () => {
const buf = new TextBuffer("abc");
// Move caret after the second character ( index 2 => after 'b' )
buf.move("right"); // -> a|bc (col 1)
buf.move("right"); // -> ab|c (col 2)
buf.backspace();
expect(buf.getLines()).toEqual(["ac"]);
expect(buf.getCursor()).toEqual([0, 1]);
});
it("merges with the previous line when invoked at column 0", () => {
const buf = new TextBuffer(["ab", "cd"].join("\n"));
// Place caret at the beginning of second line
buf.move("down"); // row = 1, col = 0
buf.backspace();
expect(buf.getLines()).toEqual(["abcd"]);
expect(buf.getCursor()).toEqual([0, 2]); // after 'b'
});
it("is a no-op at the very beginning of the buffer", () => {
const buf = new TextBuffer("ab");
buf.backspace(); // caret starts at (0,0)
expect(buf.getLines()).toEqual(["ab"]);
expect(buf.getCursor()).toEqual([0, 0]);
});
});
describe("cursor initialization", () => {
it("initializes cursor to (0,0) by default", () => {
const buf = new TextBuffer("hello\nworld");
expect(buf.getCursor()).toEqual([0, 0]);
});
it("sets cursor to valid position within line", () => {
const buf = new TextBuffer("hello", 2);
expect(buf.getCursor()).toEqual([0, 2]); // cursor at 'l'
});
it("sets cursor to end of line", () => {
const buf = new TextBuffer("hello", 5);
expect(buf.getCursor()).toEqual([0, 5]); // cursor after 'o'
});
it("sets cursor across multiple lines", () => {
const buf = new TextBuffer("hello\nworld", 7);
expect(buf.getCursor()).toEqual([1, 1]); // cursor at 'o' in 'world'
});
it("defaults to position 0 for invalid index", () => {
const buf = new TextBuffer("hello", 999);
expect(buf.getCursor()).toEqual([0, 0]);
});
});
/* ------------------------------------------------------------------ */
/* Vertical cursor movement we should preserve the preferred column */
/* ------------------------------------------------------------------ */
describe("up / down navigation keeps the preferred column", () => {
it("restores horizontal position when moving across shorter lines", () => {
// Three lines: long / short / long
const lines = ["abcdef", "x", "abcdefg"].join("\n");
const buf = new TextBuffer(lines);
// Place caret after the 5th char in first line (col = 5)
buf.move("end"); // col 6 (after 'f')
buf.move("left"); // col 5 (between 'e' and 'f')
// Move down twice through a short line and back to a long one
buf.move("down"); // should land on (1, 1) due to clamp
buf.move("down"); // desired: (2, 5)
expect(buf.getCursor()).toEqual([2, 5]);
});
});
/* ------------------------------------------------------------------ */
/* Left / Right arrow navigation across Unicode surrogate pairs */
/* ------------------------------------------------------------------ */
describe("left / right navigation", () => {
it("should treat multicodeunit emoji as a single character", () => {
// '🐶' is a surrogatepair (length 2) but one userperceived char.
const buf = new TextBuffer("🐶a");
// Move caret once to the right logically past the emoji.
buf.move("right");
// Insert another printable character
buf.insert("x");
// We expect the emoji to stay intact and the text to be 🐶xa
expect(buf.getLines()).toEqual(["🐶xa"]);
// Cursor should be after the inserted char (two visible columns along)
expect(buf.getCursor()).toEqual([0, 2]);
});
});
/* ------------------------------------------------------------------ */
/* HandleInput raw DEL bytes should map to backspace */
/* ------------------------------------------------------------------ */
it("handleInput should treat \x7f input as backspace", () => {
const buf = new TextBuffer("");
const vp = { width: 80, height: 25 };
// Type "hello" via printable input path
for (const ch of "hello") {
buf.handleInput(ch, {}, vp);
}
// Two DEL bytes terminal's backspace
buf.handleInput("\x7f", {}, vp);
buf.handleInput("\x7f", {}, vp);
expect(buf.getText()).toBe("hel");
expect(buf.getCursor()).toEqual([0, 3]);
});
/* ------------------------------------------------------------------ */
/* HandleInput `key.delete` should ALSO behave as backspace */
/* ------------------------------------------------------------------ */
it("handleInput should treat key.delete as backspace", () => {
const buf = new TextBuffer("");
const vp = { width: 80, height: 25 };
for (const ch of "hello") {
buf.handleInput(ch, {}, vp);
}
// Simulate the Delete (Mac backspace) key three times
buf.handleInput(undefined, { delete: true }, vp);
buf.handleInput(undefined, { delete: true }, vp);
buf.handleInput(undefined, { delete: true }, vp);
expect(buf.getText()).toBe("he");
expect(buf.getCursor()).toEqual([0, 2]);
});
/* ------------------------------------------------------------------ */
/* Cursor positioning semantics */
/* ------------------------------------------------------------------ */
describe("cursor movement & backspace semantics", () => {
it("typing should leave cursor after the last inserted character", () => {
const vp = { width: 80, height: 25 };
const buf = new TextBuffer("");
buf.handleInput("h", {}, vp);
expect(buf.getCursor()).toEqual([0, 1]);
for (const ch of "ello") {
buf.handleInput(ch, {}, vp);
}
expect(buf.getCursor()).toEqual([0, 5]); // after 'o'
});
it("arrowleft moves the caret to *between* characters (highlight next)", () => {
const vp = { width: 80, height: 25 };
const buf = new TextBuffer("");
for (const ch of "bar") {
buf.handleInput(ch, {}, vp);
} // cursor at col 3
buf.move("left"); // col 2 (right before 'r')
buf.move("left"); // col 1 (right before 'a')
expect(buf.getCursor()).toEqual([0, 1]);
// Character to the RIGHT of caret should be 'a'
const charRight = [...buf.getLines()[0]!][buf.getCursor()[1]];
expect(charRight).toBe("a");
// Backspace should delete the char to the *left* (i.e. 'b'), leaving "ar"
buf.backspace();
expect(buf.getLines()[0]).toBe("ar");
expect(buf.getCursor()).toEqual([0, 0]);
});
});
});