From 73fe1381aa56f249b7c511fb033e7548c14d9100 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 12 May 2025 13:38:10 -0700 Subject: [PATCH] chore: introduce new --native flag to Node module release process (#844) This PR introduces an optional build flag, `--native`, that will build a version of the Codex npm module that: - Includes both the Node.js and native Rust versions (for Mac and Linux) - Will run the native version if `CODEX_RUST=1` is set - Runs the TypeScript version otherwise Note this PR also updates the workflow URL to https://github.com/openai/codex/actions/runs/14872557396, as that is a build from today that includes everything up through https://github.com/openai/codex/pull/843. Test Plan: In `~/code/codex/codex-cli`, I ran: ``` pnpm stage-release --native ``` The end of the output was: ``` Staged version 0.1.2505121317 for release in /var/folders/wm/f209bc1n2bd_r0jncn9s6j_00000gp/T/tmp.xd2p5ETYGN Test Node: node /var/folders/wm/f209bc1n2bd_r0jncn9s6j_00000gp/T/tmp.xd2p5ETYGN/bin/codex.js --help Test Rust: CODEX_RUST=1 node /var/folders/wm/f209bc1n2bd_r0jncn9s6j_00000gp/T/tmp.xd2p5ETYGN/bin/codex.js --help Next: cd "/var/folders/wm/f209bc1n2bd_r0jncn9s6j_00000gp/T/tmp.xd2p5ETYGN" && npm publish --tag native ``` I verified that running each of these commands ran the expected version of Codex. While here, I also added `bin` to the `files` list in `package.json`, which should have been done as part of https://github.com/openai/codex/pull/757, as that added new entries to `bin` that were matched by `.gitignore` but should have been included in a release. --- README.md | 18 ++- codex-cli/.eslintrc.cjs | 2 +- codex-cli/bin/codex.js | 83 +++++++++++- codex-cli/package.json | 1 + codex-cli/scripts/install_native_deps.sh | 64 ++++++++-- codex-cli/scripts/stage_release.sh | 155 ++++++++++++++++++++--- 6 files changed, 277 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 53a9718c..eaccfebc 100644 --- a/README.md +++ b/README.md @@ -652,17 +652,21 @@ The **DCO check** blocks merges until every commit in the PR carries the footer ### Releasing `codex` -To publish a new version of the CLI, run the following in the `codex-cli` folder to stage the release in a temporary directory: +To publish a new version of the CLI you first need to stage the npm package. A +helper script in `codex-cli/scripts/` does all the heavy lifting. Inside the +`codex-cli` folder run: -``` +```bash +# Classic, JS implementation that includes small, native binaries for Linux sandboxing. pnpm stage-release -``` -Note you can specify the folder for the staged release: - -``` +# Optionally specify the temp directory to reuse between runs. RELEASE_DIR=$(mktemp -d) -pnpm stage-release "$RELEASE_DIR" +pnpm stage-release --tmp "$RELEASE_DIR" + +# "Fat" package that additionally bundles the native Rust CLI binaries for +# Linux. End-users can then opt-in at runtime by setting CODEX_RUST=1. +pnpm stage-release --native ``` Go to the folder where the release is staged and verify that it works as intended. If so, run the following from the temp folder: diff --git a/codex-cli/.eslintrc.cjs b/codex-cli/.eslintrc.cjs index b376b109..a623d2ed 100644 --- a/codex-cli/.eslintrc.cjs +++ b/codex-cli/.eslintrc.cjs @@ -1,6 +1,6 @@ module.exports = { root: true, - env: { browser: true, es2020: true }, + env: { browser: true, node: true, es2020: true }, extends: [ "eslint:recommended", "plugin:@typescript-eslint/recommended", diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index 1df18d1f..818b3627 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -1,17 +1,89 @@ #!/usr/bin/env node +// Unified entry point for the Codex CLI. +/* + * Behavior + * ========= + * 1. By default we import the JavaScript implementation located in + * dist/cli.js. + * + * 2. Developers can opt-in to a pre-compiled Rust binary by setting the + * environment variable CODEX_RUST to a truthy value (`1`, `true`, etc.). + * When that variable is present we resolve the correct binary for the + * current platform / architecture and execute it via child_process. + * + * If the CODEX_RUST=1 is specified and there is no native binary for the + * current platform / architecture, an error is thrown. + */ -// Unified entry point for Codex CLI on all platforms -// Dynamically loads the compiled ESM bundle in dist/cli.js +import { spawnSync } from "child_process"; +import path from "path"; +import { fileURLToPath, pathToFileURL } from "url"; -import path from 'path'; -import { fileURLToPath, pathToFileURL } from 'url'; +// Determine whether the user explicitly wants the Rust CLI. +const wantsNative = + process.env.CODEX_RUST != null + ? ["1", "true", "yes"].includes(process.env.CODEX_RUST.toLowerCase()) + : false; + +// Try native binary if requested. +if (wantsNative) { + const { platform, arch } = process; + + let targetTriple = null; + switch (platform) { + case "linux": + switch (arch) { + case "x64": + targetTriple = "x86_64-unknown-linux-musl"; + break; + case "arm64": + targetTriple = "aarch64-unknown-linux-gnu"; + break; + default: + break; + } + break; + case "darwin": + switch (arch) { + case "x64": + targetTriple = "x86_64-apple-darwin"; + break; + case "arm64": + targetTriple = "aarch64-apple-darwin"; + break; + default: + break; + } + break; + default: + break; + } + + if (!targetTriple) { + throw new Error(`Unsupported platform: ${platform} (${arch})`); + } + + // __dirname equivalent in ESM + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + + const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); + const result = spawnSync(binaryPath, process.argv.slice(2), { + stdio: "inherit", + }); + + const exitCode = typeof result.status === "number" ? result.status : 1; + process.exit(exitCode); +} + +// Fallback: execute the original JavaScript CLI. // Determine this script's directory const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); // Resolve the path to the compiled CLI bundle -const cliPath = path.resolve(__dirname, '../dist/cli.js'); +const cliPath = path.resolve(__dirname, "../dist/cli.js"); const cliUrl = pathToFileURL(cliPath).href; // Load and execute the CLI @@ -21,7 +93,6 @@ const cliUrl = pathToFileURL(cliPath).href; } catch (err) { // eslint-disable-next-line no-console console.error(err); - // eslint-disable-next-line no-undef process.exit(1); } })(); diff --git a/codex-cli/package.json b/codex-cli/package.json index e2454582..524e4065 100644 --- a/codex-cli/package.json +++ b/codex-cli/package.json @@ -23,6 +23,7 @@ "stage-release": "./scripts/stage_release.sh" }, "files": [ + "bin", "dist" ], "dependencies": { diff --git a/codex-cli/scripts/install_native_deps.sh b/codex-cli/scripts/install_native_deps.sh index 2b2768af..07dd73bc 100755 --- a/codex-cli/scripts/install_native_deps.sh +++ b/codex-cli/scripts/install_native_deps.sh @@ -1,20 +1,44 @@ -#!/bin/bash +#!/usr/bin/env bash -# Copy the Linux sandbox native binaries into the bin/ subfolder of codex-cli/. +# Install native runtime dependencies for codex-cli. # -# Usage: -# ./scripts/install_native_deps.sh [CODEX_CLI_ROOT] +# By default the script copies the sandbox binaries that are required at +# runtime. When called with the --full-native flag, it additionally +# bundles pre-built Rust CLI binaries so that the resulting npm package can run +# the native implementation when users set CODEX_RUST=1. # -# Arguments -# [CODEX_CLI_ROOT] – Optional. If supplied, it should be the codex-cli -# folder that contains the package.json for @openai/codex. +# Usage +# install_native_deps.sh [RELEASE_ROOT] [--full-native] # -# When no argument is given we assume the script is being run directly from a -# development checkout. In that case we install the binaries into the -# repository’s own `bin/` directory so that the CLI can run locally. +# The optional RELEASE_ROOT is the path that contains package.json. Omitting +# it installs the binaries into the repository's own bin/ folder to support +# local development. set -euo pipefail +# ------------------ +# Parse arguments +# ------------------ + +DEST_DIR="" +INCLUDE_RUST=0 + +for arg in "$@"; do + case "$arg" in + --full-native) + INCLUDE_RUST=1 + ;; + *) + if [[ -z "$DEST_DIR" ]]; then + DEST_DIR="$arg" + else + echo "Unexpected argument: $arg" >&2 + exit 1 + fi + ;; + esac +done + # ---------------------------------------------------------------------------- # Determine where the binaries should be installed. # ---------------------------------------------------------------------------- @@ -41,7 +65,7 @@ mkdir -p "$BIN_DIR" # Until we start publishing stable GitHub releases, we have to grab the binaries # from the GitHub Action that created them. Update the URL below to point to the # appropriate workflow run: -WORKFLOW_URL="https://github.com/openai/codex/actions/runs/14763725716" +WORKFLOW_URL="https://github.com/openai/codex/actions/runs/14950726936" WORKFLOW_ID="${WORKFLOW_URL##*/}" ARTIFACTS_DIR="$(mktemp -d)" @@ -50,12 +74,26 @@ trap 'rm -rf "$ARTIFACTS_DIR"' EXIT # NB: The GitHub CLI `gh` must be installed and authenticated. gh run download --dir "$ARTIFACTS_DIR" --repo openai/codex "$WORKFLOW_ID" -# Decompress the two target architectures. +# Decompress the artifacts for Linux sandboxing. zstd -d "$ARTIFACTS_DIR/x86_64-unknown-linux-musl/codex-linux-sandbox-x86_64-unknown-linux-musl.zst" \ -o "$BIN_DIR/codex-linux-sandbox-x64" zstd -d "$ARTIFACTS_DIR/aarch64-unknown-linux-gnu/codex-linux-sandbox-aarch64-unknown-linux-gnu.zst" \ -o "$BIN_DIR/codex-linux-sandbox-arm64" -echo "Installed native dependencies into $BIN_DIR" +if [[ "$INCLUDE_RUST" -eq 1 ]]; then + # x64 Linux + zstd -d "$ARTIFACTS_DIR/x86_64-unknown-linux-musl/codex-x86_64-unknown-linux-musl.zst" \ + -o "$BIN_DIR/codex-x86_64-unknown-linux-musl" + # ARM64 Linux + zstd -d "$ARTIFACTS_DIR/aarch64-unknown-linux-gnu/codex-aarch64-unknown-linux-gnu.zst" \ + -o "$BIN_DIR/codex-aarch64-unknown-linux-gnu" + # x64 macOS + zstd -d "$ARTIFACTS_DIR/x86_64-apple-darwin/codex-x86_64-apple-darwin.zst" \ + -o "$BIN_DIR/codex-x86_64-apple-darwin" + # ARM64 macOS + zstd -d "$ARTIFACTS_DIR/aarch64-apple-darwin/codex-aarch64-apple-darwin.zst" \ + -o "$BIN_DIR/codex-aarch64-apple-darwin" +fi +echo "Installed native dependencies into $BIN_DIR" diff --git a/codex-cli/scripts/stage_release.sh b/codex-cli/scripts/stage_release.sh index e92b1131..fb641d35 100755 --- a/codex-cli/scripts/stage_release.sh +++ b/codex-cli/scripts/stage_release.sh @@ -1,28 +1,145 @@ -#!/bin/bash +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# stage_release.sh +# ----------------------------------------------------------------------------- +# Stages an npm release for @openai/codex. +# +# The script used to accept a single optional positional argument that indicated +# the temporary directory in which to stage the package. We now support a +# flag-based interface so that we can extend the command with further options +# without breaking the call-site contract. +# +# --tmp : Use instead of a freshly created temp directory. +# --native : Bundle the pre-built Rust CLI binaries for Linux alongside +# the JavaScript implementation (a so-called "fat" package). +# -h|--help : Print usage. +# +# When --native is supplied we copy the linux-sandbox binaries (as before) and +# additionally fetch / unpack the two Rust targets that we currently support: +# - x86_64-unknown-linux-musl +# - aarch64-unknown-linux-gnu +# +# NOTE: This script is intended to be run from the repository root via +# `pnpm --filter codex-cli stage-release ...` or inside codex-cli with the +# helper script entry in package.json (`pnpm stage-release ...`). +# ----------------------------------------------------------------------------- set -euo pipefail -# Change to the codex-cli directory. -cd "$(dirname "${BASH_SOURCE[0]}")/.." +# Helper - usage / flag parsing -# First argument is where to stage the release. Creates a temporary directory -# if not provided. -RELEASE_DIR="${1:-$(mktemp -d)}" -[ -n "${1-}" ] && shift +usage() { + cat <&2 + usage 1 + ;; + *) + echo "Unexpected extra argument: $1" >&2 + usage 1 + ;; + esac + shift +done + +# Fallback when the caller did not specify a directory. +# If no directory was specified create a fresh temporary one. +if [[ -z "$TMPDIR" ]]; then + TMPDIR="$(mktemp -d)" +fi + +# Ensure the directory exists, then resolve to an absolute path. +mkdir -p "$TMPDIR" +TMPDIR="$(cd "$TMPDIR" && pwd)" + +# Main build logic + +echo "Staging release in $TMPDIR" + +# The script lives in codex-cli/scripts/ - change into codex-cli root so that +# relative paths keep working. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +pushd "$CODEX_CLI_ROOT" >/dev/null + +# 1. Build the JS artifacts --------------------------------------------------- -# Compile the JavaScript. pnpm install pnpm build -mkdir "$RELEASE_DIR/bin" -cp -r bin/codex.js "$RELEASE_DIR/bin/codex.js" -cp -r dist "$RELEASE_DIR/dist" -cp -r src "$RELEASE_DIR/src" # important if we want sourcemaps to continue to work -cp ../README.md "$RELEASE_DIR" -# TODO: Derive version from Git tag. -VERSION=$(printf '0.1.%d' "$(date +%y%m%d%H%M)") -jq --arg version "$VERSION" '.version = $version' package.json > "$RELEASE_DIR/package.json" -# Copy the native dependencies. -./scripts/install_native_deps.sh "$RELEASE_DIR" +# Paths inside the staged package +mkdir -p "$TMPDIR/bin" -echo "Staged version $VERSION for release in $RELEASE_DIR" +cp -r bin/codex.js "$TMPDIR/bin/codex.js" +cp -r dist "$TMPDIR/dist" +cp -r src "$TMPDIR/src" # keep source for TS sourcemaps +cp ../README.md "$TMPDIR" || true # README is one level up - ignore if missing + +# Derive a timestamp-based version (keep same scheme as before) +VERSION="$(printf '0.1.%d' "$(date +%y%m%d%H%M)")" + +# Modify package.json - bump version and optionally add the native directory to +# the files array so that the binaries are published to npm. + +jq --arg version "$VERSION" \ + '.version = $version' \ + package.json > "$TMPDIR/package.json" + +# 2. Native runtime deps (sandbox plus optional Rust binaries) + +if [[ "$INCLUDE_NATIVE" -eq 1 ]]; then + ./scripts/install_native_deps.sh "$TMPDIR" --full-native +else + ./scripts/install_native_deps.sh "$TMPDIR" +fi + +popd >/dev/null + +echo "Staged version $VERSION for release in $TMPDIR" + +echo "Test Node:" +echo " node ${TMPDIR}/bin/codex.js --help" +if [[ "$INCLUDE_NATIVE" -eq 1 ]]; then + echo "Test Rust:" + echo " CODEX_RUST=1 node ${TMPDIR}/bin/codex.js --help" +fi + +# Print final hint for convenience +if [[ "$INCLUDE_NATIVE" -eq 1 ]]; then + echo "Next: cd \"$TMPDIR\" && npm publish --tag native" +else + echo "Next: cd \"$TMPDIR\" && npm publish" +fi