fix(cli): honor token controls for builtin documents (#7897)

This commit is contained in:
morluto authored and GitHub committed 2026-10-29 08:11:19 +08:00
1 parent b0b25ad93e
commit c7722c8eeb
2 files changed
+103 -7

No files matched your search

+39 -7
View File
@@ -158,6 +158,7 @@ export const createCliResultOutput = (
let failed = false;
let error: AnalysisErrorProjection | undefined;
let streaming = false;
let sdkDocumentControls = false;
const sdkParts: string[] = [];
let sdkCompletion: Promise<void> | undefined;
let completed:
@@ -266,21 +267,34 @@ export const createCliResultOutput = (
error = undefined;
completed = undefined;
streaming = false;
sdkDocumentControls = false;
sdkParts.length = 0;
sdkCompletion = undefined;
},
sdkArguments(args) {
const normalized = normalizeCliArguments(args);
const split = splitIncurBuiltinArguments(normalized);
const values = new Set(split.outputValues.map(({ index }) => index));
const hasFlag = (flag: string) =>
normalized.some((token, index) => token === flag && !values.has(index));
// Incur emits these documents outside command result handlers. Manifest
// flags precede help; schema follows help and version in its dispatcher.
sdkDocumentControls =
(options.tokenCount || options.tokenWindow) &&
(hasFlag("--llms") ||
hasFlag("--llms-full") ||
(hasFlag("--schema") &&
!["completions", "mcp", "skills"].includes(
split.command[0] ?? "",
) &&
!hasFlag("--help") &&
!hasFlag("-h") &&
!hasFlag("--version")));
// The caller's parsed controls remain authoritative: the CLI may adapt
// CSV/TSV to JSON for Incur without changing the requested presentation.
if (format !== "jsonl" || !options.tokenCount) return normalized;
// Incur counts errors before REA can retain their diagnostic payload.
// Preserve operands using the shared builtin consumption rules.
const values = new Set(
splitIncurBuiltinArguments(normalized).outputValues.map(
({ index }) => index,
),
);
return normalized.filter(
(token, index) => token !== "--token-count" || values.has(index),
);
@@ -289,7 +303,8 @@ export const createCliResultOutput = (
streaming = true;
},
captureSdkOutput(text) {
if (format !== "jsonl" || streaming || handled) return false;
if ((format !== "jsonl" && !sdkDocumentControls) || streaming || handled)
return false;
// Incur already materializes one-shot documents. Retain references only;
// generator events bypass this buffer and remain incremental.
if (text.length > 0) sdkParts.push(text);
@@ -312,11 +327,28 @@ export const createCliResultOutput = (
destination,
`${estimateTokenCount(parts.join(""))}\n`,
);
else
else if (sdkDocumentControls && options.tokenWindow) {
const text = parts.join("");
const window = sliceTextByTokenWindow(
text,
options.tokenOffset ?? 0,
options.tokenLimit,
);
await writeOutputText(
destination,
window.truncated
? `${window.text}\n[truncated: showing tokens ${window.start}–${window.end} of ${window.total}]\n`
: text,
);
} else
for (const part of parts)
await writeOutputText(destination, part);
} else {
const text = parts.join("");
if (format !== "jsonl") {
await writeOutputText(destination, text);
return;
}
let result: unknown = text;
try {
result = JSON.parse(text);
@@ -0,0 +1,64 @@
import { estimateTokenCount } from "tokenx";
import { expect } from "vitest";
import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js";
workspaceCliTest(
"applies token controls to SDK manifest and schema documents",
async ({ cli }) => {
const run = async (...arguments_: string[]) => {
const result = await cli.run({ arguments: arguments_ });
expect(result.exitCode, result.stderr).toBe(0);
expect(result.stderr).toBe("");
return result.stdout;
};
for (const args of [
["--llms"],
["--llms-full"],
["--llms-full", "--format", "json"],
["inspect-devicetree", "--schema"],
["inspect-devicetree", "--schema", "--format", "json"],
]) {
const document = await run(...args);
expect(await run(...args, "--token-count")).toBe(
`${estimateTokenCount(document)}\n`,
);
}
const document = await run("--llms-full");
const page = await run("--llms-full", "--token-limit", "1");
expect(page.length).toBeLessThan(document.length);
expect(page).toMatch(/\[truncated: showing tokens 0–1 of \d+\]\n$/);
expect(await run("--llms-full", "--token-limit", "1000000")).toBe(document);
expect(
await run("inspect-devicetree", "--schema", "--token-limit", "1"),
).toMatch(/\[truncated: showing tokens 0–1 of \d+\]\n$/);
// Schema does not outrank help or plain version in Incur's dispatch.
expect(
await run("inspect-devicetree", "--schema", "--help", "--token-count"),
).toBe(await run("inspect-devicetree", "--schema", "--help"));
expect(await run("--schema", "--version", "--token-count")).toBe(
await run("--schema", "--version"),
);
expect(await run("completions", "bash", "--schema", "--token-count")).toBe(
await run("completions", "bash", "--schema"),
);
const unknown = await cli.run({
arguments: ["no-such-command", "--schema"],
});
const unknownCount = await cli.run({
arguments: ["no-such-command", "--schema", "--token-count"],
});
expect(unknown.exitCode).toBe(1);
expect(unknownCount.exitCode).toBe(1);
expect(unknownCount.stdout).toBe(unknown.stdout);
const structuredWindow = await cli.run({
arguments: ["--llms-full", "--format", "json", "--token-limit", "1"],
});
expect(structuredWindow.exitCode).toBe(1);
expect(structuredWindow.stdout).toContain("UNSUPPORTED_OUTPUT_COMBINATION");
},
);