From c7722c8eebea60ce3399961bc63b226f59876edd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:11:19 +0800 Subject: [PATCH] fix(cli): honor token controls for builtin documents (#7897) --- src/cli/cliResultOutput.ts | 46 +++++++++++-- .../cliBuiltinDocumentTokens.test.ts | 64 +++++++++++++++++++ 2 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 tests/acceptance/cliBuiltinDocumentTokens.test.ts diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index ea29d27cbd..d3bde08fb4 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -158,6 +158,7 @@ export const createCliResultOutput = ( let failed = false; let error: AnalysisErrorProjection | undefined; let streaming = false; + let sdkDocumentControls = false; const sdkParts: string[] = []; let sdkCompletion: Promise | undefined; let completed: @@ -266,21 +267,34 @@ export const createCliResultOutput = ( error = undefined; completed = undefined; streaming = false; + sdkDocumentControls = false; sdkParts.length = 0; sdkCompletion = undefined; }, sdkArguments(args) { const normalized = normalizeCliArguments(args); + const split = splitIncurBuiltinArguments(normalized); + const values = new Set(split.outputValues.map(({ index }) => index)); + const hasFlag = (flag: string) => + normalized.some((token, index) => token === flag && !values.has(index)); + // Incur emits these documents outside command result handlers. Manifest + // flags precede help; schema follows help and version in its dispatcher. + sdkDocumentControls = + (options.tokenCount || options.tokenWindow) && + (hasFlag("--llms") || + hasFlag("--llms-full") || + (hasFlag("--schema") && + !["completions", "mcp", "skills"].includes( + split.command[0] ?? "", + ) && + !hasFlag("--help") && + !hasFlag("-h") && + !hasFlag("--version"))); // The caller's parsed controls remain authoritative: the CLI may adapt // CSV/TSV to JSON for Incur without changing the requested presentation. if (format !== "jsonl" || !options.tokenCount) return normalized; // Incur counts errors before REA can retain their diagnostic payload. // Preserve operands using the shared builtin consumption rules. - const values = new Set( - splitIncurBuiltinArguments(normalized).outputValues.map( - ({ index }) => index, - ), - ); return normalized.filter( (token, index) => token !== "--token-count" || values.has(index), ); @@ -289,7 +303,8 @@ export const createCliResultOutput = ( streaming = true; }, captureSdkOutput(text) { - if (format !== "jsonl" || streaming || handled) return false; + if ((format !== "jsonl" && !sdkDocumentControls) || streaming || handled) + return false; // Incur already materializes one-shot documents. Retain references only; // generator events bypass this buffer and remain incremental. if (text.length > 0) sdkParts.push(text); @@ -312,11 +327,28 @@ export const createCliResultOutput = ( destination, `${estimateTokenCount(parts.join(""))}\n`, ); - else + else if (sdkDocumentControls && options.tokenWindow) { + const text = parts.join(""); + const window = sliceTextByTokenWindow( + text, + options.tokenOffset ?? 0, + options.tokenLimit, + ); + await writeOutputText( + destination, + window.truncated + ? `${window.text}\n[truncated: showing tokens ${window.start}–${window.end} of ${window.total}]\n` + : text, + ); + } else for (const part of parts) await writeOutputText(destination, part); } else { const text = parts.join(""); + if (format !== "jsonl") { + await writeOutputText(destination, text); + return; + } let result: unknown = text; try { result = JSON.parse(text); diff --git a/tests/acceptance/cliBuiltinDocumentTokens.test.ts b/tests/acceptance/cliBuiltinDocumentTokens.test.ts new file mode 100644 index 0000000000..648aee5c57 --- /dev/null +++ b/tests/acceptance/cliBuiltinDocumentTokens.test.ts @@ -0,0 +1,64 @@ +import { estimateTokenCount } from "tokenx"; +import { expect } from "vitest"; + +import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js"; + +workspaceCliTest( + "applies token controls to SDK manifest and schema documents", + async ({ cli }) => { + const run = async (...arguments_: string[]) => { + const result = await cli.run({ arguments: arguments_ }); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + return result.stdout; + }; + + for (const args of [ + ["--llms"], + ["--llms-full"], + ["--llms-full", "--format", "json"], + ["inspect-devicetree", "--schema"], + ["inspect-devicetree", "--schema", "--format", "json"], + ]) { + const document = await run(...args); + expect(await run(...args, "--token-count")).toBe( + `${estimateTokenCount(document)}\n`, + ); + } + + const document = await run("--llms-full"); + const page = await run("--llms-full", "--token-limit", "1"); + expect(page.length).toBeLessThan(document.length); + expect(page).toMatch(/\[truncated: showing tokens 0–1 of \d+\]\n$/); + expect(await run("--llms-full", "--token-limit", "1000000")).toBe(document); + expect( + await run("inspect-devicetree", "--schema", "--token-limit", "1"), + ).toMatch(/\[truncated: showing tokens 0–1 of \d+\]\n$/); + + // Schema does not outrank help or plain version in Incur's dispatch. + expect( + await run("inspect-devicetree", "--schema", "--help", "--token-count"), + ).toBe(await run("inspect-devicetree", "--schema", "--help")); + expect(await run("--schema", "--version", "--token-count")).toBe( + await run("--schema", "--version"), + ); + expect(await run("completions", "bash", "--schema", "--token-count")).toBe( + await run("completions", "bash", "--schema"), + ); + + const unknown = await cli.run({ + arguments: ["no-such-command", "--schema"], + }); + const unknownCount = await cli.run({ + arguments: ["no-such-command", "--schema", "--token-count"], + }); + expect(unknown.exitCode).toBe(1); + expect(unknownCount.exitCode).toBe(1); + expect(unknownCount.stdout).toBe(unknown.stdout); + const structuredWindow = await cli.run({ + arguments: ["--llms-full", "--format", "json", "--token-limit", "1"], + }); + expect(structuredWindow.exitCode).toBe(1); + expect(structuredWindow.stdout).toContain("UNSUPPORTED_OUTPUT_COMBINATION"); + }, +);