diff --git a/docs/cli.md b/docs/cli.md index b8122f7076..1bf3908e1c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -592,10 +592,17 @@ redirected, which keeps large results smaller for agents and scripts; pipe through `jq .` for indented files. Output selection and formatting do not change operation status. +`--format md` escapes backslashes and pipes in table headers and cells, and +renders cell line breaks as `
`. These are presentation changes; retained +Evidence and JSON output keep the original values. Table widths are measured +iteratively, so an admitted row page is not limited by the JavaScript function +argument count. + `--filter-output` selects fields from successful command data. Classified failures retain their complete typed diagnostic projection even when the selector matches a field such as `code` or `message`, or names a missing field. `--full-output` controls whether the outer `{ok, error, meta}` envelope is included. +Selected own JSON members, including `__proto__`, remain ordinary output data. Sibling array slices that select different fields are combined by original row index and emitted in source order, regardless of selector order. Selecting a diff --git a/src/cli/cliOutputFormatter.ts b/src/cli/cliOutputFormatter.ts new file mode 100644 index 0000000000..51e8a0bf78 --- /dev/null +++ b/src/cli/cliOutputFormatter.ts @@ -0,0 +1,150 @@ +/* + * Markdown output layout is adapted from Incur's Formatter. + * Copyright (c) 2025-present weth, LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +import { Formatter } from "incur"; + +/** Keep REA's Markdown layout compatible with Incur while owning table safety. */ +export const formatCliOutput = ( + value: unknown, + format: Formatter.Format, +): string => + format === "md" ? formatMarkdown(value) : Formatter.format(value, format); + +const isScalar = (value: unknown): boolean => + value === null || value === undefined || typeof value !== "object"; + +const isFlatObject = (value: Record): boolean => + Object.values(value).every(isScalar); + +const isObjectArray = (value: unknown): value is Record[] => + Array.isArray(value) && + value.length > 0 && + value.every( + (item) => typeof item === "object" && item !== null && !Array.isArray(item), + ); + +const escapeTableCell = (value: string): string => + value + .replace(/\\/g, "\\\\") + .replace(/\|/g, "\\|") + .replace(/\r\n|\r|\n/g, "
"); + +const renderTable = ( + headers: readonly string[], + rows: readonly string[][], +): string => { + const escapedHeaders = headers.map(escapeTableCell); + const widths = escapedHeaders.map((header) => header.length); + for (const row of rows) { + for (let index = 0; index < escapedHeaders.length; index += 1) { + const width = escapeTableCell(row[index] ?? "").length; + if (width > widths[index]!) widths[index] = width; + } + } + + const pad = (value: string, index: number): string => + value.padEnd(widths[index]!); + const lines = [ + `| ${escapedHeaders.map(pad).join(" | ")} |`, + `|${widths.map((width) => "-".repeat(width + 2)).join("|")}|`, + ]; + for (const row of rows) { + lines.push( + `| ${escapedHeaders + .map((_, index) => pad(escapeTableCell(row[index] ?? ""), index)) + .join(" | ")} |`, + ); + } + return lines.join("\n"); +}; + +const formatKeyValueTable = (value: Record): string => + renderTable( + ["Key", "Value"], + Object.entries(value).map(([key, child]) => [key, String(child)]), + ); + +const formatObjectTable = ( + items: readonly Record[], +): string => { + const headers: string[] = []; + const seen = new Set(); + for (const item of items) + for (const key of Object.keys(item)) + if (!seen.has(key)) { + seen.add(key); + headers.push(key); + } + return renderTable( + headers, + items.map((item) => headers.map((key) => String(item[key] ?? ""))), + ); +}; + +const formatMarkdown = ( + value: unknown, + path: readonly string[] = [], +): string => { + if (value == null && path.length === 0) return ""; + if (isScalar(value)) { + if (path.length === 0) return String(value); + return `## ${path.join(".")}\n\n${String(value)}`; + } + if (Array.isArray(value)) { + if (isObjectArray(value)) { + const table = formatObjectTable(value); + return path.length === 0 ? table : `## ${path.join(".")}\n\n${table}`; + } + return formatMarkdown(String(value), path); + } + + const object = value as Record; + const entries = Object.entries(object); + if (path.length === 0 && isFlatObject(object)) + return formatKeyValueTable(object); + + const needsHeadings = + path.length > 0 || + entries.length > 1 || + entries.some(([, child]) => !isScalar(child)); + if (!needsHeadings) return formatKeyValueTable(object); + + return entries + .map(([key, child]) => { + const childPath = [...path, key]; + const heading = `## ${childPath.join(".")}\n\n`; + if (isScalar(child)) return `${heading}${String(child)}`; + if (isObjectArray(child)) return `${heading}${formatObjectTable(child)}`; + if ( + typeof child === "object" && + child !== null && + !Array.isArray(child) + ) { + const nested = child as Record; + return isFlatObject(nested) + ? `${heading}${formatKeyValueTable(nested)}` + : formatMarkdown(nested, childPath); + } + return `${heading}${String(child)}`; + }) + .join("\n\n"); +}; diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 921106b866..26541a4fdb 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -1,7 +1,7 @@ import { constants } from "node:buffer"; import type { Writable } from "node:stream"; -import { Filter, Formatter } from "incur"; +import { Filter } from "incur"; import { estimateTokenCount } from "tokenx"; import { @@ -18,6 +18,7 @@ import { projectAnalysisError, type AnalysisErrorProjection, } from "../domain/analysisErrorProjection.js"; +import { formatCliOutput } from "./cliOutputFormatter.js"; import { jsonOutputParts, writeJsonOutput, @@ -164,7 +165,7 @@ export const createCliResultOutput = ( : error; // Recovery must remain actionable even when a token window/count was // requested. Do not apply those controls to this small typed failure. - const text = Formatter.format(document, format); + const text = formatCliOutput(document, format); handled = true; try { await writeOutputText(destination, `${text}\n`); @@ -247,9 +248,10 @@ export const createCliResultOutput = ( !options.tokenCount && format !== "json" && format !== "jsonl"; - // Delegate unfiltered successful non-JSON rendering to Incur. This boundary - // owns filtered successes, requested failure projections, and token windows. + // Delegate other human formats to Incur. Markdown stays here so admission + // and emitted table text use the same safe formatter. if ( + format !== "md" && format !== "json" && format !== "jsonl" && operationFailure === undefined && @@ -280,7 +282,7 @@ export const createCliResultOutput = ( } } if (characters <= constants.MAX_STRING_LENGTH) { - const count = estimateTokenCount(Formatter.format(tokenValue, format)); + const count = estimateTokenCount(formatCliOutput(tokenValue, format)); handled = true; try { await writeOutputText(destination, `${count}\n`); @@ -334,7 +336,7 @@ export const createCliResultOutput = ( ? {} : filtered; if (ownsSuccessfulTokenWindow) { - const formatted = Formatter.format(filtered, format); + const formatted = formatCliOutput(filtered, format); const window = sliceTextByTokenWindow( formatted, options.tokenOffset ?? 0, @@ -349,7 +351,7 @@ export const createCliResultOutput = ( ? `${window.text}\n[truncated: showing tokens ${options.tokenOffset ?? 0}–${window.end} of ${window.total}]` : formatted; const outputDocument = options.fullOutput - ? Formatter.format( + ? formatCliOutput( window.truncated ? { ok: true, @@ -383,11 +385,20 @@ export const createCliResultOutput = ( // Once writing starts, a failed destination must not receive a second // document from Incur's error formatter. const formattedText = - (operationFailure !== undefined || ownsFilteredOutput) && - format !== "json" && - format !== "jsonl" - ? Formatter.format(document, format) + format === "md" || + ((operationFailure !== undefined || ownsFilteredOutput) && + format !== "json" && + format !== "jsonl") + ? formatCliOutput(document, format) : undefined; + if ( + format === "md" && + formattedText === "" && + !options.fullOutput && + !ownsFilteredOutput && + operationFailure === undefined + ) + return false; handled = true; try { if (formattedText !== undefined) { @@ -422,7 +433,7 @@ const fullFailureEnvelope = ( ): Readonly> => { if (tokenLimit === undefined && tokenOffset === undefined) return { ok: false, error: failure, meta }; - const formatted = Formatter.format(failure, format); + const formatted = formatCliOutput(failure, format); const offset = tokenOffset ?? 0; const window = sliceTextByTokenWindow(formatted, offset, tokenLimit); if (!window.truncated) return { ok: false, error: failure, meta }; @@ -449,7 +460,12 @@ const applyCliOutputFilter = (value: unknown, paths: FilterPaths): unknown => { "key" in paths[0][0] ) { const key = paths[0][0].key; - if (typeof value !== "object" || value === null) return undefined; + if ( + typeof value !== "object" || + value === null || + !Object.hasOwn(value, key) + ) + return undefined; const selected = (value as Record)[key]; return typeof selected !== "object" || selected === null ? selected @@ -470,31 +486,32 @@ const projectObjectPaths = (value: unknown, paths: FilterPaths): unknown => { byKey.set(segment.key, siblings); } - const result: Record = {}; + const result = new Map(); for (const key of [...byKey.keys()].sort()) { const siblings = byKey.get(key)!; + if (!Object.hasOwn(value, key)) continue; const child = (value as Record)[key]; if (child === undefined) continue; // Selecting a parent retains its complete original subtree and dominates // any narrower selections below it, independent of path order. if (siblings.some((path) => path.length === 0)) { - result[key] = child; + result.set(key, child); continue; } const slicePaths = siblings.filter( (path) => path[0] !== undefined && "start" in path[0], ); if (slicePaths.length > 0 && Array.isArray(child)) { - result[key] = projectArraySlices(child, slicePaths); + result.set(key, projectArraySlices(child, slicePaths)); continue; } const objectPaths = siblings.filter( (path) => path[0] !== undefined && "key" in path[0], ); if (objectPaths.length > 0 && typeof child === "object" && child !== null) - result[key] = projectObjectPaths(child, objectPaths); + result.set(key, projectObjectPaths(child, objectPaths)); } - return result; + return Object.fromEntries(result); }; /** Union sliced rows by their original index, then emit them in source order. */ diff --git a/tests/boundary/cli/analysisViewCli.test.ts b/tests/boundary/cli/analysisViewCli.test.ts index 271df8b80c..816526f118 100644 --- a/tests/boundary/cli/analysisViewCli.test.ts +++ b/tests/boundary/cli/analysisViewCli.test.ts @@ -552,6 +552,92 @@ cliTest( expect(complete.json).toEqual(bundle); }, ); + +cliTest( + "formats the full caller-admitted native inventory page as Markdown and JSON", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-native-inventory-markdown-", + ); + const input = join(root, "view.json"); + const rowCount = 150_000; + { + const parent = createEvidence( + { + path: "/fixtures/native.exe", + format: "pe", + sha256: "a".repeat(64), + }, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "list_names", + parameters: {}, + result: Array.from({ length: rowCount }, (_, index) => ({ + address: `0x${index.toString(16)}`, + value: `item${index}`, + })), + }, + ); + const request = { + source: { kind: "inline", evidence: parent }, + view: { + kind: "native-inventory", + limit: rowCount, + fields: ["value"], + }, + }; + await writeFile(input, JSON.stringify(request)); + } + const environment = { + HOME: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + REA_LOG_LEVEL: "silent", + }; + + { + const json = await cli.run({ + arguments: [ + "inspect-analysis-view", + input, + "--format", + "json", + "--filter-output", + "normalized_result.rows", + ], + environment, + timeoutMs: 60_000, + }); + expect(json.exitCode).toBe(0); + expect( + (json.json as { normalized_result: { rows: unknown[] } }) + .normalized_result.rows, + ).toHaveLength(rowCount); + } + + const markdown = await cli.run({ + arguments: [ + "inspect-analysis-view", + input, + "--format", + "md", + "--filter-output", + "normalized_result.rows", + ], + environment, + timeoutMs: 60_000, + }); + expect(markdown.exitCode).toBe(0); + expect(markdown.stdout).toContain("## normalized_result.rows"); + expect(markdown.stdout).toContain("| source_index"); + let lineBreaks = 0; + for (const character of markdown.stdout) + if (character === "\n") lineBreaks += 1; + expect(lineBreaks).toBeGreaterThan(rowCount); + expect(markdown.stdout).toContain("149999"); + }, +); + cliTest( "projects one section from a JSON file of inline layout Evidence", async ({ cli }) => { diff --git a/tests/boundary/cli/evidenceBundleCommands.test.ts b/tests/boundary/cli/evidenceBundleCommands.test.ts index a0e890fbfa..3883e7dba9 100644 --- a/tests/boundary/cli/evidenceBundleCommands.test.ts +++ b/tests/boundary/cli/evidenceBundleCommands.test.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { expect } from "vitest"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; import { @@ -147,3 +148,75 @@ cliTest( } }, ); + +cliTest( + "preserves a selected own __proto__ member from complete bundle data", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-evidence-filter-proto-"); + const path = join(root, "bundle.json"); + const result = jsonValueSchema.parse( + JSON.parse('{"__proto__":{"answer":7},"normal":{"answer":7}}'), + ); + const record = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { operation: "ordinary-json", parameters: {}, result }, + ); + await writeFile(path, JSON.stringify(createEvidenceBundle([record]))); + + const output = await cli.run({ + arguments: [ + "inspect-evidence-bundle", + JSON.stringify({ path, detail: "complete" }), + "--json", + "--filter-output", + "records[0,1].normalized_result.__proto__.answer", + ], + }); + + expect(output.exitCode).toBe(0); + const records = ( + output.json as { + records: Array<{ normalized_result: unknown }>; + } + ).records; + const normalizedResult = records[0]?.normalized_result; + expect(typeof normalizedResult).toBe("object"); + expect(normalizedResult).not.toBeNull(); + expect(Object.hasOwn(normalizedResult as object, "__proto__")).toBe(true); + expect((normalizedResult as Record)["__proto__"]).toEqual({ + answer: 7, + }); + }, +); + +cliTest( + "preserves cell text when formatting retained Evidence as Markdown", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-evidence-markdown-cells-"); + const path = join(root, "bundle.json"); + const result = { text: "left|right\\tail\nnext" }; + const record = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "ordinary-json", + parameters: {}, + result, + }, + ); + await writeFile(path, JSON.stringify(createEvidenceBundle([record]))); + const base = [ + "inspect-evidence-bundle", + JSON.stringify({ path, detail: "complete" }), + "--filter-output", + "records.0.normalized_result", + ]; + const json = await cli.run({ arguments: [...base, "--json"] }); + expect(json.exitCode, json.stderr).toBe(0); + expect(json.stdout).toContain(JSON.stringify(result.text)); + const markdown = await cli.run({ arguments: [...base, "--format", "md"] }); + expect(markdown.exitCode, markdown.stderr).toBe(0); + expect(markdown.stdout).toContain("left\\|right\\\\tail
next"); + }, +); diff --git a/tests/boundary/cli/filteredArrayOutput.test.ts b/tests/boundary/cli/filteredArrayOutput.test.ts index 726727dd2c..c9ae33ffa2 100644 --- a/tests/boundary/cli/filteredArrayOutput.test.ts +++ b/tests/boundary/cli/filteredArrayOutput.test.ts @@ -210,6 +210,35 @@ it("finishes JSONL after a bounded large record before a missing selection", asy expect(output.failed).toBe(false); }); +it("does not invent selected JSON members from the object prototype", async () => { + const value = freezeRecursively({ selected: { present: 7 } }); + const output = await runFilteredCli(value, "json", "selected.__proto__"); + expect(JSON.parse(output)).toEqual({ selected: {} }); + expect(value).toEqual({ selected: { present: 7 } }); +}); + +it("escapes table delimiters and line breaks in Markdown cells", async () => { + const value = freezeRecursively({ + selected: { + pipe: "left|right", + backslash: "left\\right", + lines: "first line\nsecond line", + }, + }); + + const output = await runFilteredCli(value, "md", "selected"); + + expect(output).toContain("left\\|right"); + expect(output).toContain("left\\\\right"); + expect(output).toContain("first line
second line"); + expect(output).not.toContain("left|right"); + expect(value.selected).toEqual({ + pipe: "left|right", + backslash: "left\\right", + lines: "first line\nsecond line", + }); +}); + const projectionSource = freezeRecursively([ { rows: [ diff --git a/tests/boundary/cli/formattingRecovery.test.ts b/tests/boundary/cli/formattingRecovery.test.ts index 5228e87ae3..698f931a70 100644 --- a/tests/boundary/cli/formattingRecovery.test.ts +++ b/tests/boundary/cli/formattingRecovery.test.ts @@ -120,6 +120,37 @@ describe("CLI formatting stage admission", () => { }); describe("CLI formatting recovery outcomes", () => { + it("formats a recovered Markdown result through the CLI Markdown owner", async () => { + const cause = new RangeError("Invalid string length"); + const chunks: Buffer[] = []; + const output = createCliResultOutput( + ["--format", "md"], + new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + ); + const value = new Proxy( + {}, + { + ownKeys: () => { + throw cause; + }, + }, + ); + + expect(await output?.write(value, { ...metadata, format: "md" })).toBe( + true, + ); + expect(output?.error).toMatchObject({ code: "resource_constraint" }); + const text = Buffer.concat(chunks).toString("utf8"); + expect(text).toContain("resource_constraint"); + expect(text).toContain("original result was not written"); + expect(output?.failed).toBe(false); + }); + it("retains ownership if the recovery document cannot be written", async () => { const cause = new RangeError("Invalid string length"); const output = createCliResultOutput(