mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 11:43:20 +00:00
test(cli): cover terminal stdout behaviour in the CLI parity goldens (#11677)
* test(cli): cover terminal stdout behaviour in the CLI parity goldens * test(cli): run the terminal command as the owned process and relay its output live
This commit is contained in:
1 parent
81fe44d7d8
commit
80c60ac68c
14 files changed
+367
-16
No files matched your search
@@ -2,7 +2,7 @@ import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { afterAll, beforeAll, describe, expect } from "vitest";
|
||||
|
||||
import { PRODUCT_IDENTITY } from "../../../src/identity.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
import { cliTest, type TestCli } from "../../support/cli/cliFixture.js";
|
||||
import {
|
||||
createTestWorkspace,
|
||||
removeTestWorkspace,
|
||||
@@ -33,6 +33,10 @@ interface ParityCase {
|
||||
readonly arguments: (workspace: TestWorkspace) => readonly string[];
|
||||
/** Shell completion protocol requests. */
|
||||
readonly complete?: "bash" | "zsh" | "fish" | "nushell";
|
||||
/** Run with stdout on a pseudo-terminal, as a user's terminal would. */
|
||||
readonly terminal?: boolean;
|
||||
/** Environment beyond the workspace's for this case. */
|
||||
readonly environment?: (workspace: TestWorkspace) => NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
const sample = (workspace: TestWorkspace) => workspace.path("sample.bin");
|
||||
@@ -60,6 +64,48 @@ const completion = (
|
||||
arguments: fixed("--", "rea", ...words),
|
||||
});
|
||||
|
||||
/**
|
||||
* Terminal ("human") cases. incur treats a terminal stdout as a person
|
||||
* reading: help instead of an error for a bare command, errors followed by
|
||||
* usage, agent-only results (setup) hidden, and an update notice. Notices are
|
||||
* off except in the notice case, whose fresh cache names a newer version so
|
||||
* no registry check is scheduled.
|
||||
*/
|
||||
const terminal = (
|
||||
name: string,
|
||||
arguments_: ParityCase["arguments"],
|
||||
environment: ParityCase["environment"] = () => ({ NO_UPDATE_NOTIFIER: "1" }),
|
||||
): ParityCase => ({
|
||||
name: `terminal-${name}`,
|
||||
terminal: true,
|
||||
arguments: arguments_,
|
||||
environment,
|
||||
});
|
||||
const UPDATE_CACHE = "update-cache";
|
||||
const NEWER_VERSION = "999.0.0";
|
||||
|
||||
const TERMINAL_CASES: readonly ParityCase[] = [
|
||||
terminal("help-root-bare", fixed()),
|
||||
terminal("help-command", fixed("inspect-byte-entropy", "--help")),
|
||||
terminal("output", (w) => entropy(w, ...WINDOW)),
|
||||
terminal("output-explicit-format", (w) =>
|
||||
entropy(w, ...WINDOW, "--format", "json"),
|
||||
),
|
||||
terminal("output-full-output", (w) => entropy(w, ...WINDOW, "--full-output")),
|
||||
terminal("error-unknown-command", fixed("no-such-command")),
|
||||
terminal("error-unknown-flag", (w) => entropy(w, "--no-such-flag")),
|
||||
terminal("error-refined-number", (w) => entropy(w, "--block-size", "100")),
|
||||
terminal("error-refined-number-json", (w) =>
|
||||
entropy(w, "--block-size", "100", "--json"),
|
||||
),
|
||||
terminal("setup-dry-run", fixed("setup", "--dry-run")),
|
||||
terminal("update-notice", fixed("no-such-command"), (w) => ({
|
||||
XDG_CACHE_HOME: w.path(UPDATE_CACHE),
|
||||
// A refresh, if one were scheduled, stays on this host.
|
||||
npm_config_registry: "http://127.0.0.1:9/",
|
||||
})),
|
||||
];
|
||||
|
||||
const CASES: readonly ParityCase[] = [
|
||||
// Help and manifests
|
||||
{ name: "help-root-bare", arguments: fixed() },
|
||||
@@ -364,6 +410,8 @@ const normalize = (text: string, workspace: TestWorkspace): string =>
|
||||
.replace(/^\d+\.\d+\.\d+(?:-[\w.]+)?\n$/u, (line) =>
|
||||
line === `${PRODUCT_IDENTITY.packageVersion}\n` ? "<version>\n" : line,
|
||||
)
|
||||
.split(`from ${PRODUCT_IDENTITY.packageVersion} to`)
|
||||
.join("from <version> to")
|
||||
.replace(/(duration"?:\s*"?)\d+ms/gu, "$1<ms>ms")
|
||||
.replace(/("command_ms":)\d+/gu, "$1<ms>"),
|
||||
);
|
||||
@@ -380,11 +428,12 @@ const render = (
|
||||
? JSON.stringify(argument)
|
||||
: argument,
|
||||
);
|
||||
const stdout = testCase.name.startsWith("help-root")
|
||||
const stdout = /^(?:terminal-)?help-root/u.test(testCase.name)
|
||||
? elideRootCommandRows(result.stdout)
|
||||
: result.stdout;
|
||||
return [
|
||||
`$ ${testCase.complete === undefined ? "" : `COMPLETE=${testCase.complete} `}rea ${command.join(" ")}`.trimEnd(),
|
||||
`$ ${testCase.complete === undefined ? "" : `COMPLETE=${testCase.complete} `}rea ${command.join(" ")}`.trimEnd() +
|
||||
(testCase.terminal === true ? " # stdout is a terminal" : ""),
|
||||
`exit: ${result.exitCode}`,
|
||||
"--- stdout",
|
||||
`${normalize(stdout, workspace)}<end>`,
|
||||
@@ -404,6 +453,10 @@ describe("CLI parity goldens", () => {
|
||||
beforeAll(async () => {
|
||||
workspace = await createTestWorkspace("rea-cli-parity-");
|
||||
await workspace.write("sample.bin", SAMPLE);
|
||||
await workspace.write(
|
||||
`${UPDATE_CACHE}/incur/updates/${PRODUCT_IDENTITY.packageName}.json`,
|
||||
`${JSON.stringify({ checkedAt: Date.now(), latest: NEWER_VERSION })}\n`,
|
||||
);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await removeTestWorkspace(workspace.root);
|
||||
@@ -414,23 +467,27 @@ describe("CLI parity goldens", () => {
|
||||
const files = (await readdir(GOLDEN_DIRECTORY)).filter((file) =>
|
||||
file.endsWith(".txt"),
|
||||
);
|
||||
const expected = CASES.map(({ name }) => `${name}.txt`).sort();
|
||||
const expected = [...CASES, ...TERMINAL_CASES]
|
||||
.map(({ name }) => `${name}.txt`)
|
||||
.sort();
|
||||
if (RECORD)
|
||||
expect(files.filter((file) => !expected.includes(file))).toEqual([]);
|
||||
else expect(files.sort()).toEqual(expected);
|
||||
});
|
||||
|
||||
cliTest.for(CASES)("$name", async (testCase, { cli }) => {
|
||||
const check = async (testCase: ParityCase, cli: TestCli): Promise<void> => {
|
||||
const result = await cli.run({
|
||||
arguments: testCase.arguments(workspace),
|
||||
cwd: workspace.root,
|
||||
environment: {
|
||||
...workspace.environment,
|
||||
...testCase.environment?.(workspace),
|
||||
...(testCase.complete === undefined
|
||||
? {}
|
||||
: { COMPLETE: testCase.complete }),
|
||||
},
|
||||
...(testCase.complete === undefined ? {} : { freshProcess: true }),
|
||||
...(testCase.terminal === true ? { terminal: true } : {}),
|
||||
});
|
||||
const actual = render(testCase, workspace, result);
|
||||
const golden = new URL(`${testCase.name}.txt`, GOLDEN_DIRECTORY);
|
||||
@@ -440,5 +497,12 @@ describe("CLI parity goldens", () => {
|
||||
return;
|
||||
}
|
||||
expect(actual).toBe(await readFile(golden, "utf8"));
|
||||
});
|
||||
};
|
||||
|
||||
cliTest.for(CASES)("$name", (testCase, { cli }) => check(testCase, cli));
|
||||
// Windows has no POSIX pseudo-terminal for the terminal driver.
|
||||
cliTest.skipIf(process.platform === "win32").for(TERMINAL_CASES)(
|
||||
"$name",
|
||||
(testCase, { cli }) => check(testCase, cli),
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
$ rea inspect-byte-entropy <workspace>/sample.bin --block-size 100 --json # stdout is a terminal
|
||||
exit: 1
|
||||
--- stdout
|
||||
{
|
||||
"code": "VALIDATION_ERROR",
|
||||
"message": "REA could not read the command arguments. Check the named argument and its expected type, then try again.",
|
||||
"fieldErrors": [
|
||||
{
|
||||
"code": "too_small",
|
||||
"missing": false,
|
||||
"path": "blockSize",
|
||||
"expected": "",
|
||||
"received": "",
|
||||
"message": "Too small: expected number to be >=256"
|
||||
},
|
||||
{
|
||||
"code": "custom",
|
||||
"missing": false,
|
||||
"path": "blockSize",
|
||||
"expected": "",
|
||||
"received": "",
|
||||
"message": "Block size must be a power of two."
|
||||
}
|
||||
]
|
||||
}
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,32 @@
|
||||
$ rea inspect-byte-entropy <workspace>/sample.bin --block-size 100 # stdout is a terminal
|
||||
exit: 1
|
||||
--- stdout
|
||||
Error: invalid value for --block-size: Too small: expected number to be >=256
|
||||
Error: invalid value for --block-size: Block size must be a power of two.
|
||||
See below for usage.
|
||||
|
||||
rea inspect-byte-entropy — Inspect file-wide and block byte entropy without executing the file
|
||||
|
||||
Usage: rea inspect-byte-entropy <path> [options]
|
||||
|
||||
Arguments:
|
||||
path Local file
|
||||
|
||||
Options:
|
||||
--block-size <value> Power-of-two block size from 256 to 65536 bytes (default: 4096)
|
||||
--offset <value> Window start as a decimal file byte offset (default: 0)
|
||||
--length <value> Window length in bytes; omit to extend through EOF
|
||||
|
||||
Global Options:
|
||||
--filter-output <keys> Filter output by key paths (e.g. foo,bar.baz,a[0,3])
|
||||
--format <toon|json|yaml|md|jsonl> Output format
|
||||
--full-output Show full output envelope
|
||||
--help Show help
|
||||
--llms, --llms-full Print LLM-readable manifest
|
||||
--schema Show JSON Schema for command
|
||||
--token-count Print token count of output (instead of output)
|
||||
--token-limit <n> Limit output to n tokens
|
||||
--token-offset <n> Skip first n tokens of output
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,11 @@
|
||||
$ rea no-such-command # stdout is a terminal
|
||||
exit: 1
|
||||
--- stdout
|
||||
Error: 'no-such-command' is not a command for 'rea'. Did you mean 'cutter-command'?
|
||||
|
||||
Suggested commands:
|
||||
rea cutter-command
|
||||
rea --help # see all available commands
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,7 @@
|
||||
$ rea inspect-byte-entropy <workspace>/sample.bin --no-such-flag # stdout is a terminal
|
||||
exit: 1
|
||||
--- stdout
|
||||
Error: Unknown flag: --no-such-flag
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,31 @@
|
||||
$ rea inspect-byte-entropy --help # stdout is a terminal
|
||||
exit: 0
|
||||
--- stdout
|
||||
rea inspect-byte-entropy — Inspect file-wide and block byte entropy without executing the file
|
||||
|
||||
Usage: rea inspect-byte-entropy <path> [options]
|
||||
|
||||
Arguments:
|
||||
path Local file
|
||||
|
||||
Options:
|
||||
--block-size <value> Power-of-two block size from 256 to 65536 bytes (default: 4096)
|
||||
--offset <value> Window start as a decimal file byte offset (default: 0)
|
||||
--length <value> Window length in bytes; omit to extend through EOF
|
||||
|
||||
Custom Global Options:
|
||||
--timings Write measured command timing to stderr.
|
||||
|
||||
Global Options:
|
||||
--filter-output <keys> Filter output by key paths (e.g. foo,bar.baz,a[0,3])
|
||||
--format <toon|json|yaml|md|jsonl> Output format
|
||||
--full-output Show full output envelope
|
||||
--help Show help
|
||||
--llms, --llms-full Print LLM-readable manifest
|
||||
--schema Show JSON Schema for command
|
||||
--token-count Print token count of output (instead of output)
|
||||
--token-limit <n> Limit output to n tokens
|
||||
--token-offset <n> Skip first n tokens of output
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,43 @@
|
||||
$ rea # stdout is a terminal
|
||||
exit: 0
|
||||
--- stdout
|
||||
rea@<version> — Reverse engineer anything from your terminal or agent.
|
||||
|
||||
Usage: rea <command>
|
||||
|
||||
Commands:
|
||||
address-to-file-offset Map one analysis address to a file offset; memory bytes may differ and ranges may not be contiguous
|
||||
<registered command rows elided>
|
||||
build-reconstruction-obligation-ledger Generate a deterministic Evidence-backed reconstruction obligation ledger page
|
||||
<registered command rows elided>
|
||||
decompile Read one part of an app as code. Supply at least one of the positional address or --procedure. Alternatively, use --file-offset alone. Equal positional and --procedure values are accepted; conflicts fail. With Ghidra, a space-qualified address such as ram:0x... avoids file-offset fallback; inspect-native-load-image reports the observed default space.
|
||||
<registered command rows elided>
|
||||
inspect-byte-entropy Inspect file-wide and block byte entropy without executing the file
|
||||
<registered command rows elided>
|
||||
setup Configure agent integrations and optional analysis providers
|
||||
<registered command rows elided>
|
||||
xrefs List bounded, untyped references to an analyzed address. For typed incoming references, use inspect-native-instruction at the target address where supported. Supply at least one of the positional address or --address. Equal values are accepted; conflicts fail.
|
||||
|
||||
Integrations:
|
||||
completions Generate shell completion script
|
||||
mcp Register as MCP server (add, doctor)
|
||||
|
||||
Custom Global Options:
|
||||
--timings Write measured command timing to stderr.
|
||||
|
||||
Global Options:
|
||||
--filter-output <keys> Filter output by key paths (e.g. foo,bar.baz,a[0,3])
|
||||
--format <toon|json|yaml|md|jsonl> Output format
|
||||
--full-output Show full output envelope
|
||||
--help Show help
|
||||
--llms, --llms-full Print LLM-readable manifest
|
||||
--mcp Start as MCP stdio server
|
||||
--schema Show JSON Schema for command
|
||||
--token-count Print token count of output (instead of output)
|
||||
--token-limit <n> Limit output to n tokens
|
||||
--token-offset <n> Skip first n tokens of output
|
||||
--update Update to latest version
|
||||
--version Show version
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,14 @@
|
||||
$ rea inspect-byte-entropy <workspace>/sample.bin --filter-output normalized_result.window --format json # stdout is a terminal
|
||||
exit: 0
|
||||
--- stdout
|
||||
{
|
||||
"normalized_result": {
|
||||
"window": {
|
||||
"offset": 0,
|
||||
"length": 20
|
||||
}
|
||||
}
|
||||
}
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,15 @@
|
||||
$ rea inspect-byte-entropy <workspace>/sample.bin --filter-output normalized_result.window --full-output # stdout is a terminal
|
||||
exit: 0
|
||||
--- stdout
|
||||
ok: true
|
||||
data:
|
||||
normalized_result:
|
||||
window:
|
||||
offset: 0
|
||||
length: 20
|
||||
meta:
|
||||
command: inspect-byte-entropy
|
||||
duration: <ms>ms
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
$ rea inspect-byte-entropy <workspace>/sample.bin --filter-output normalized_result.window # stdout is a terminal
|
||||
exit: 0
|
||||
--- stdout
|
||||
normalized_result:
|
||||
window:
|
||||
offset: 0
|
||||
length: 20
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -0,0 +1,14 @@
|
||||
$ rea setup --dry-run # stdout is a terminal
|
||||
exit: 0
|
||||
--- stdout
|
||||
<end>
|
||||
--- stderr
|
||||
┌ REA setup plan
|
||||
│
|
||||
◆ Ready to review
|
||||
│
|
||||
│ REA will preserve unrelated configuration and will not install or upgrade Node.js, npm, Homebrew, Java, or Ghidra.
|
||||
│
|
||||
└ Dry run complete. No changes were made.
|
||||
|
||||
<end>
|
||||
@@ -0,0 +1,12 @@
|
||||
$ rea no-such-command # stdout is a terminal
|
||||
exit: 1
|
||||
--- stdout
|
||||
Error: 'no-such-command' is not a command for 'rea'. Did you mean 'cutter-command'?
|
||||
|
||||
Suggested commands:
|
||||
rea cutter-command
|
||||
rea --help # see all available commands
|
||||
rea --update # upgrade from <version> to 999.0.0
|
||||
<end>
|
||||
--- stderr
|
||||
<end>
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
createCliCommandSession,
|
||||
type CliCommandSession,
|
||||
} from "./cliCommandSession.js";
|
||||
import { terminalStdoutInvocation } from "./terminalStdout.js";
|
||||
|
||||
/** Caller-controlled values for one compiled REA CLI invocation. */
|
||||
export interface CliInvocation {
|
||||
@@ -34,6 +35,12 @@ export interface CliInvocation {
|
||||
* command. Use it for process-level facts such as startup or exit.
|
||||
*/
|
||||
readonly freshProcess?: boolean;
|
||||
/**
|
||||
* Put stdout on a pseudo-terminal (POSIX only), as when a user runs `rea`
|
||||
* in a terminal. Stdin is empty and stderr stays a pipe. Always starts a
|
||||
* fresh process.
|
||||
*/
|
||||
readonly terminal?: boolean;
|
||||
}
|
||||
|
||||
/** Normalized result from the compiled REA CLI. */
|
||||
@@ -279,8 +286,10 @@ const sharedProcessCanServe = ({
|
||||
input,
|
||||
timeoutMs,
|
||||
freshProcess,
|
||||
terminal,
|
||||
}: CliInvocation): boolean =>
|
||||
freshProcess !== true &&
|
||||
terminal !== true &&
|
||||
input === undefined &&
|
||||
timeoutMs === undefined &&
|
||||
arguments_[0] !== "mcp" &&
|
||||
@@ -318,17 +327,21 @@ export const createTestCli = (
|
||||
environment,
|
||||
input,
|
||||
timeoutMs,
|
||||
terminal,
|
||||
}) => {
|
||||
const result = await processes.run(
|
||||
process.execPath,
|
||||
[resolve("scripts/rea.mjs"), ...arguments_],
|
||||
{
|
||||
cwd: cwd ?? process.cwd(),
|
||||
env: cliProcessEnvironment(environment, compileCache),
|
||||
...(input === undefined ? {} : { input }),
|
||||
...(timeoutMs === undefined ? {} : { timeoutMs }),
|
||||
},
|
||||
);
|
||||
if (terminal === true && input !== undefined)
|
||||
throw new Error("A terminal run has an empty stdin.");
|
||||
const rea = [resolve("scripts/rea.mjs"), ...arguments_];
|
||||
const { command, arguments: commandArguments } =
|
||||
terminal === true
|
||||
? terminalStdoutInvocation(process.execPath, rea)
|
||||
: { command: process.execPath, arguments: rea };
|
||||
const result = await processes.run(command, commandArguments, {
|
||||
cwd: cwd ?? process.cwd(),
|
||||
env: cliProcessEnvironment(environment, compileCache),
|
||||
...(input === undefined ? {} : { input }),
|
||||
...(timeoutMs === undefined ? {} : { timeoutMs }),
|
||||
});
|
||||
return { ...result, json: parseJson(result.stdout) };
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Run a command with stdout on a pseudo-terminal, so it sees
|
||||
* `process.stdout.isTTY` as a user's terminal would, while stdin stays empty
|
||||
* and stderr stays a separate pipe. Python's standard `pty` support provides
|
||||
* the terminal without a native Node dependency.
|
||||
*
|
||||
* The driver forks a relay that copies the terminal to its stdout as bytes
|
||||
* arrive, then replaces itself with the command. The process the caller
|
||||
* starts is therefore the command itself: its deadline signals (SIGTERM,
|
||||
* then SIGKILL) reach the command, and output written before a timeout is
|
||||
* already captured. The relay ends when the command's terminal closes.
|
||||
* Output post-processing is off, so the captured bytes are the ones the
|
||||
* command wrote, and the window is a fixed 80x24 so width-dependent output
|
||||
* does not follow the host.
|
||||
*/
|
||||
const DRIVER = String.raw`
|
||||
import fcntl, os, struct, sys, termios
|
||||
|
||||
leader, follower = os.openpty()
|
||||
attributes = termios.tcgetattr(follower)
|
||||
attributes[1] &= ~termios.OPOST
|
||||
termios.tcsetattr(follower, termios.TCSANOW, attributes)
|
||||
fcntl.ioctl(follower, termios.TIOCSWINSZ, struct.pack("HHHH", 24, 80, 0, 0))
|
||||
if os.fork() == 0:
|
||||
os.close(follower)
|
||||
while True:
|
||||
try:
|
||||
chunk = os.read(leader, 65536)
|
||||
except OSError:
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
view = memoryview(chunk)
|
||||
while view:
|
||||
view = view[os.write(1, view):]
|
||||
os._exit(0)
|
||||
os.close(leader)
|
||||
empty = os.open(os.devnull, os.O_RDONLY)
|
||||
os.dup2(empty, 0)
|
||||
os.dup2(follower, 1)
|
||||
os.close(empty)
|
||||
os.close(follower)
|
||||
os.execv(sys.argv[1], sys.argv[1:])
|
||||
`;
|
||||
|
||||
/** The command and arguments that run `command` with a terminal stdout. */
|
||||
export const terminalStdoutInvocation = (
|
||||
command: string,
|
||||
arguments_: readonly string[],
|
||||
): { readonly command: string; readonly arguments: readonly string[] } => {
|
||||
if (process.platform === "win32")
|
||||
throw new Error("A terminal stdout needs a POSIX pseudo-terminal.");
|
||||
return {
|
||||
command: "python3",
|
||||
arguments: ["-I", "-S", "-c", DRIVER, command, ...arguments_],
|
||||
};
|
||||
};
|
||||
Reference in new issue
Block a user