mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 14:05:49 +00:00
fix(cli): honor token controls for builtin documents (#7897)
This commit is contained in:
1 parent
b0b25ad93e
commit
c7722c8eeb
2 files changed
+103
-7
No files matched your search
@@ -158,6 +158,7 @@ export const createCliResultOutput = (
|
||||
let failed = false;
|
||||
let error: AnalysisErrorProjection | undefined;
|
||||
let streaming = false;
|
||||
let sdkDocumentControls = false;
|
||||
const sdkParts: string[] = [];
|
||||
let sdkCompletion: Promise<void> | undefined;
|
||||
let completed:
|
||||
@@ -266,21 +267,34 @@ export const createCliResultOutput = (
|
||||
error = undefined;
|
||||
completed = undefined;
|
||||
streaming = false;
|
||||
sdkDocumentControls = false;
|
||||
sdkParts.length = 0;
|
||||
sdkCompletion = undefined;
|
||||
},
|
||||
sdkArguments(args) {
|
||||
const normalized = normalizeCliArguments(args);
|
||||
const split = splitIncurBuiltinArguments(normalized);
|
||||
const values = new Set(split.outputValues.map(({ index }) => index));
|
||||
const hasFlag = (flag: string) =>
|
||||
normalized.some((token, index) => token === flag && !values.has(index));
|
||||
// Incur emits these documents outside command result handlers. Manifest
|
||||
// flags precede help; schema follows help and version in its dispatcher.
|
||||
sdkDocumentControls =
|
||||
(options.tokenCount || options.tokenWindow) &&
|
||||
(hasFlag("--llms") ||
|
||||
hasFlag("--llms-full") ||
|
||||
(hasFlag("--schema") &&
|
||||
!["completions", "mcp", "skills"].includes(
|
||||
split.command[0] ?? "",
|
||||
) &&
|
||||
!hasFlag("--help") &&
|
||||
!hasFlag("-h") &&
|
||||
!hasFlag("--version")));
|
||||
// The caller's parsed controls remain authoritative: the CLI may adapt
|
||||
// CSV/TSV to JSON for Incur without changing the requested presentation.
|
||||
if (format !== "jsonl" || !options.tokenCount) return normalized;
|
||||
// Incur counts errors before REA can retain their diagnostic payload.
|
||||
// Preserve operands using the shared builtin consumption rules.
|
||||
const values = new Set(
|
||||
splitIncurBuiltinArguments(normalized).outputValues.map(
|
||||
({ index }) => index,
|
||||
),
|
||||
);
|
||||
return normalized.filter(
|
||||
(token, index) => token !== "--token-count" || values.has(index),
|
||||
);
|
||||
@@ -289,7 +303,8 @@ export const createCliResultOutput = (
|
||||
streaming = true;
|
||||
},
|
||||
captureSdkOutput(text) {
|
||||
if (format !== "jsonl" || streaming || handled) return false;
|
||||
if ((format !== "jsonl" && !sdkDocumentControls) || streaming || handled)
|
||||
return false;
|
||||
// Incur already materializes one-shot documents. Retain references only;
|
||||
// generator events bypass this buffer and remain incremental.
|
||||
if (text.length > 0) sdkParts.push(text);
|
||||
@@ -312,11 +327,28 @@ export const createCliResultOutput = (
|
||||
destination,
|
||||
`${estimateTokenCount(parts.join(""))}\n`,
|
||||
);
|
||||
else
|
||||
else if (sdkDocumentControls && options.tokenWindow) {
|
||||
const text = parts.join("");
|
||||
const window = sliceTextByTokenWindow(
|
||||
text,
|
||||
options.tokenOffset ?? 0,
|
||||
options.tokenLimit,
|
||||
);
|
||||
await writeOutputText(
|
||||
destination,
|
||||
window.truncated
|
||||
? `${window.text}\n[truncated: showing tokens ${window.start}–${window.end} of ${window.total}]\n`
|
||||
: text,
|
||||
);
|
||||
} else
|
||||
for (const part of parts)
|
||||
await writeOutputText(destination, part);
|
||||
} else {
|
||||
const text = parts.join("");
|
||||
if (format !== "jsonl") {
|
||||
await writeOutputText(destination, text);
|
||||
return;
|
||||
}
|
||||
let result: unknown = text;
|
||||
try {
|
||||
result = JSON.parse(text);
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { estimateTokenCount } from "tokenx";
|
||||
import { expect } from "vitest";
|
||||
|
||||
import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js";
|
||||
|
||||
workspaceCliTest(
|
||||
"applies token controls to SDK manifest and schema documents",
|
||||
async ({ cli }) => {
|
||||
const run = async (...arguments_: string[]) => {
|
||||
const result = await cli.run({ arguments: arguments_ });
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
expect(result.stderr).toBe("");
|
||||
return result.stdout;
|
||||
};
|
||||
|
||||
for (const args of [
|
||||
["--llms"],
|
||||
["--llms-full"],
|
||||
["--llms-full", "--format", "json"],
|
||||
["inspect-devicetree", "--schema"],
|
||||
["inspect-devicetree", "--schema", "--format", "json"],
|
||||
]) {
|
||||
const document = await run(...args);
|
||||
expect(await run(...args, "--token-count")).toBe(
|
||||
`${estimateTokenCount(document)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const document = await run("--llms-full");
|
||||
const page = await run("--llms-full", "--token-limit", "1");
|
||||
expect(page.length).toBeLessThan(document.length);
|
||||
expect(page).toMatch(/\[truncated: showing tokens 0–1 of \d+\]\n$/);
|
||||
expect(await run("--llms-full", "--token-limit", "1000000")).toBe(document);
|
||||
expect(
|
||||
await run("inspect-devicetree", "--schema", "--token-limit", "1"),
|
||||
).toMatch(/\[truncated: showing tokens 0–1 of \d+\]\n$/);
|
||||
|
||||
// Schema does not outrank help or plain version in Incur's dispatch.
|
||||
expect(
|
||||
await run("inspect-devicetree", "--schema", "--help", "--token-count"),
|
||||
).toBe(await run("inspect-devicetree", "--schema", "--help"));
|
||||
expect(await run("--schema", "--version", "--token-count")).toBe(
|
||||
await run("--schema", "--version"),
|
||||
);
|
||||
expect(await run("completions", "bash", "--schema", "--token-count")).toBe(
|
||||
await run("completions", "bash", "--schema"),
|
||||
);
|
||||
|
||||
const unknown = await cli.run({
|
||||
arguments: ["no-such-command", "--schema"],
|
||||
});
|
||||
const unknownCount = await cli.run({
|
||||
arguments: ["no-such-command", "--schema", "--token-count"],
|
||||
});
|
||||
expect(unknown.exitCode).toBe(1);
|
||||
expect(unknownCount.exitCode).toBe(1);
|
||||
expect(unknownCount.stdout).toBe(unknown.stdout);
|
||||
const structuredWindow = await cli.run({
|
||||
arguments: ["--llms-full", "--format", "json", "--token-limit", "1"],
|
||||
});
|
||||
expect(structuredWindow.exitCode).toBe(1);
|
||||
expect(structuredWindow.stdout).toContain("UNSUPPORTED_OUTPUT_COMBINATION");
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user