diff --git a/docs/getting-started/cli.md b/docs/getting-started/cli.md index f8839696ae..2f8dc5323c 100644 --- a/docs/getting-started/cli.md +++ b/docs/getting-started/cli.md @@ -1488,7 +1488,20 @@ descendants in other process groups that inherited its run token. If that cleanup cannot be confirmed, the report keeps its earlier checks and adds a failed `cleanup` check with the reason and retained run identifier. -The default terminal format is TOON. Use `--json` when saving results for a +The default terminal format is TOON. Complete TOON results are validated +before stdout receives the document. Large results use a private file in the +system temporary directory until delivery finishes and need free space there. +If closing or removing that stage fails, CLI shutdown retains and retries its +cleanup in order. The reported cleanup failure preserves any original formatting +or temporary-storage refusal. SIGINT/SIGTERM remain owned through staging, +delivery and cleanup: pending I/O settles before further writes stop and the +stage is removed. +A document can exceed the runtime's string limit when each encoded line fits. +Staging limits the encoded text kept in memory. The TOON encoder still +normalizes the input tree before yielding lines, so working memory also +depends on the size of the input and the longest encoded line. +Token counting and token windows still require a complete formatted string. +Use `--json` to stream the result without TOON staging or to save it for a JSON consumer. Valued output flags (`--format`, `--filter-output`, `--token-limit`, `--token-offset`) accept `--flag value` or `--flag=value`; a missing value fails with `MISSING_OUTPUT_VALUE` before the command runs. diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 5a5e45478b..c3031a965a 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -38,6 +38,12 @@ import { formatCliCallToAction, type CallToAction } from "./outputEnvelope.js"; import { jsonLinesErrorRecord } from "./jsonLines.js"; import type { OutputFilterPath } from "./outputFilter.js"; import { writeCliJsonLines } from "./jsonLinesOutput.js"; +import { + ToonOutputCleanupFailure, + toonOutputCancellationStatus, + writeToonOutput, +} from "./toonOutput.js"; +import { analysisErrorWithCleanupDependencies } from "../domain/analysisErrorCleanup.js"; import { jsonOutputParts, isTerminalOutput, @@ -45,6 +51,8 @@ import { writeOutputParts, writeOutputText, } from "./jsonOutput.js"; +export { toonOutputCancellationStatus as cliOutputCancellationStatus } from "./toonOutput.js"; + /** Result metadata supplied by the command that actually ran. */ export interface CliResultOutputMetadata { readonly command: string; @@ -104,6 +112,7 @@ export const createCliResultOutput = ( | "tsv"; let handled = false; let failed = false; + let interrupted = false; let error: AnalysisErrorProjection | undefined; let completed: | { @@ -117,38 +126,68 @@ export const createCliResultOutput = ( const recoverFormattingFailure = async (cause: unknown): Promise => { // A rejected command never reaches write(). Destination failures retain // handled ownership. Only a completed, unwritten result can be replaced. + const formattingCause = + cause instanceof ToonOutputCleanupFailure + ? cause.outputFailure?.cause + : cause; if ( completed === undefined || handled || - hostRangeErrorKind(cause) !== "host-capacity" || - !(cause instanceof Error) || - cause.message !== "Invalid string length" + !(formattingCause instanceof Error) ) return false; + const stringCapacity = + hostRangeErrorKind(formattingCause) === "host-capacity" && + formattingCause.message === "Invalid string length"; + const systemCode = + "code" in formattingCause ? formattingCause.code : undefined; + const stagingCapacity = + format === "toon" && + (systemCode === "ENOSPC" || + systemCode === "EDQUOT" || + systemCode === "EFBIG") + ? systemCode + : undefined; + if (!stringCapacity && stagingCapacity === undefined) return false; const { originalFailure } = completed; - error = projectAnalysisError( - retainCliPublicationOnError( - new AnalysisResourceConstraintError( - completed.metadata.command, - "memory", - `The ${format} result could not fit in a runtime string. The original result was not written.`, - { - format, - formatted_characters: null, - max_string_characters: constants.MAX_STRING_LENGTH, - ...(originalFailure === undefined - ? {} - : { original_failure: originalFailure }), - }, - { - cause, - retryable: false, - remediationAction: - "Use --format json to stream the complete result, or narrow it with --filter-output. Token limits are applied after formatting and cannot avoid this constraint.", - }, - ), - completed.receipt, + const refusal = retainCliPublicationOnError( + new AnalysisResourceConstraintError( + completed.metadata.command, + stagingCapacity === undefined ? "memory" : "capacity", + stagingCapacity === undefined + ? `The ${format} result could not fit in a runtime string. The original result was not written.` + : "The temporary storage could not stage the TOON result. The original result was not written.", + { + format, + ...(stagingCapacity === undefined + ? { + formatted_characters: null, + max_string_characters: constants.MAX_STRING_LENGTH, + } + : { system_code: stagingCapacity }), + ...(originalFailure === undefined + ? {} + : { original_failure: originalFailure }), + }, + { + cause: formattingCause, + retryable: false, + remediationAction: + stagingCapacity === undefined + ? "Use --format json to stream the complete result, or narrow it with --filter-output. Token limits are applied after formatting and cannot avoid this constraint." + : "Free space in the system temporary directory, use --format json, or narrow the result with --filter-output.", + }, ), + completed.receipt, + ); + error = projectAnalysisError( + cause instanceof ToonOutputCleanupFailure + ? analysisErrorWithCleanupDependencies( + refusal, + cause, + completed.metadata.command, + ) + : refusal, ); const document = options.fullOutput ? { @@ -231,6 +270,12 @@ export const createCliResultOutput = ( } return written; } catch (cause: unknown) { + if (toonOutputCancellationStatus(cause) !== undefined) { + interrupted = true; + handled = true; + failed = true; + throw cause; + } if (await recoverFormattingFailure(cause)) return true; if (handled) failed = true; throw cause; @@ -251,11 +296,12 @@ export const createCliResultOutput = ( format = options.format as typeof format; handled = false; failed = false; + interrupted = false; error = undefined; completed = undefined; }, finish() { - return writeOutputText(destination, ""); + return interrupted ? Promise.resolve() : writeOutputText(destination, ""); }, write, recoverFormattingFailure, @@ -640,6 +686,17 @@ export const createCliResultOutput = ( } return true; } + if (format === "toon") { + await writeToonOutput( + document, + destination, + () => { + handled = true; + }, + metadata.command, + ); + return true; + } // Once writing starts, a failed destination must not receive another document. const formattedText = format === "json" ? undefined : formatCliOutput(document, format); diff --git a/src/cli/commandCancellation.ts b/src/cli/commandCancellation.ts index e9a3214551..c045610324 100644 --- a/src/cli/commandCancellation.ts +++ b/src/cli/commandCancellation.ts @@ -7,28 +7,60 @@ export interface CommandCancellationHost { exitCode?: number | string | null | undefined; } +/** Own process signals until both the operation and its resources settle. */ +export class CommandCancellationScope { + readonly #controller = new AbortController(); + #exitStatus: 130 | 143 | undefined; + #finished = false; + #closed = false; + + constructor(private readonly host: CommandCancellationHost = process) { + host.on("SIGINT", this.#interrupt); + host.on("SIGTERM", this.#terminate); + } + + get signal(): AbortSignal { + return this.#controller.signal; + } + + get exitStatus(): 130 | 143 | undefined { + return this.#exitStatus; + } + + readonly #interrupt = () => this.#abort("SIGINT", 130); + readonly #terminate = () => this.#abort("SIGTERM", 143); + + #abort(signal: "SIGINT" | "SIGTERM", status: 130 | 143): void { + this.#exitStatus ??= status; + this.#controller.abort(CallerAbort.forProcessSignal(signal)); + if (this.#finished) this.finish(); + } + + /** Publish the disposition while a failed resource close still owns signals. */ + finish(): void { + this.#finished = true; + if (this.#exitStatus !== undefined) this.host.exitCode = this.#exitStatus; + } + + /** Release signal ownership only after the resource owner confirms cleanup. */ + close(): void { + if (this.#closed) return; + this.#closed = true; + this.host.off("SIGINT", this.#interrupt); + this.host.off("SIGTERM", this.#terminate); + this.finish(); + } +} + /** Translate one-shot CLI interrupts into awaited cleanup and conventional exit codes. */ export const withCommandCancellation = async ( operation: (signal: AbortSignal) => Promise, host: CommandCancellationHost = process, ): Promise => { - const controller = new AbortController(); - let exitCode: number | undefined; - const interrupt = () => { - exitCode ??= 130; - controller.abort(CallerAbort.forProcessSignal("SIGINT")); - }; - const terminate = () => { - exitCode ??= 143; - controller.abort(CallerAbort.forProcessSignal("SIGTERM")); - }; - host.on("SIGINT", interrupt); - host.on("SIGTERM", terminate); + const scope = new CommandCancellationScope(host); try { - return await operation(controller.signal); + return await operation(scope.signal); } finally { - host.off("SIGINT", interrupt); - host.off("SIGTERM", terminate); - if (exitCode !== undefined) host.exitCode = exitCode; + scope.close(); } }; diff --git a/src/cli/toonOutput.ts b/src/cli/toonOutput.ts new file mode 100644 index 0000000000..d81e8daccf --- /dev/null +++ b/src/cli/toonOutput.ts @@ -0,0 +1,227 @@ +import { open } from "node:fs/promises"; +import { join } from "node:path"; +import type { Writable } from "node:stream"; + +import { encodeLines } from "@toon-format/toon"; + +import { + AnalysisCancelledError, + cancelledBy, +} from "../domain/analysisErrorCore.js"; +import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; +import { CommandCancellationScope } from "./commandCancellation.js"; +import { bufferedJsonParts } from "../domain/jsonSerialization.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { + AnalysisCleanupLease, + AnalysisCleanupLeaseGroup, + ArtifactResourceScope, +} from "../resources/index.js"; +import { ownCliProcessResource } from "./cliResources.js"; +import { formatCliOutput } from "./cliOutputFormatter.js"; +import { writeOutputParts, writeOutputText } from "./jsonOutput.js"; + +const retainedStages = new Set(); +ownCliProcessResource("toon-output", async () => { + const failures: unknown[] = []; + for (const resources of retainedStages) { + try { + await resources.close(); + retainedStages.delete(resources); + } catch (cause: unknown) { + failures.push(cause); + } + } + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) + throw new AggregateError(failures, "TOON staging cleanup failed"); +}); + +/** The stage owns both its original output failure and retryable cleanup. */ +export class ToonOutputCleanupFailure extends ProviderCleanupError { + constructor( + readonly outputFailure: { readonly cause: unknown } | undefined, + operation: string, + resources: readonly string[], + cleanupCause: unknown, + ) { + super( + "cli-toon-output", + resources, + { + reason: + cleanupCause instanceof Error + ? cleanupCause.message + : "Stage cleanup failed", + }, + { + operation, + cause: new AggregateError( + [ + ...(outputFailure === undefined ? [] : [outputFailure.cause]), + cleanupCause, + ], + "TOON output and cleanup failures", + ), + }, + ); + } +} + +// Only this output owner can attest a process disposition for a cancellation. +// The shared factory remains the sole constructor of the domain error. +const cancellationStatuses = new WeakMap(); + +/** Read the output owner's disposition without inferring it from ambient state. */ +export const toonOutputCancellationStatus = ( + cause: unknown, +): 130 | 143 | undefined => { + const failure = + cause instanceof ToonOutputCleanupFailure + ? cause.outputFailure?.cause + : cause; + return failure instanceof AnalysisCancelledError + ? cancellationStatuses.get(failure) + : undefined; +}; + +function* toonParts(value: object): Generator { + for (const line of encodeLines(value)) { + yield line; + yield "\n"; + } +} + +/** + * Complete encoding before releasing stdout. Keep a small document in memory + * and stage a larger one privately so a late encoding failure publishes no + * prefix. The same encoded bytes are read back without visiting the input again. + */ +export const writeToonOutput = async ( + value: unknown, + destination: Writable, + beginOutput: () => void, + operation: string, +): Promise => { + if (typeof value !== "object" || value === null) { + const text = `${formatCliOutput(value, "toon")}\n`; + beginOutput(); + await writeOutputText(destination, text); + return; + } + + const parts = bufferedJsonParts(toonParts(value)); + try { + const buffered: string[] = []; + let characters = 0; + let next = parts.next(); + while (!next.done && characters + next.value.length <= 1024 * 1024) { + buffered.push(next.value); + characters += next.value.length; + next = parts.next(); + } + if (next.done) { + beginOutput(); + await writeOutputParts( + destination, + buffered.length === 0 ? ["\n"] : buffered, + ); + return; + } + const firstStagedPart = next.value; + const cancellation = new CommandCancellationScope(); + let retained = false; + const cancelled = () => { + const status = cancellation.exitStatus; + if (status === undefined) return undefined; + const failure = cancelledBy(operation, cancellation.signal); + cancellationStatuses.set(failure, status); + return failure; + }; + const checkCancellation = () => { + const failure = cancelled(); + if (failure !== undefined) throw failure; + }; + try { + const root = await PrivateRuntimeRoot.create({ prefix: "rea-toon-" }); + const directory = root.path; + const path = join(directory, "output"); + const resources = new ArtifactResourceScope(); + let owner: OwnedFileHandle | undefined; + const lease = new AnalysisCleanupLease( + async () => { + await owner?.close(); + owner = undefined; + await root.close(); + cancellation.close(); + }, + { + get cleanupResources() { + return owner === undefined ? [directory] : [path, directory]; + }, + }, + ); + let failure: { cause: unknown } | undefined; + try { + checkCancellation(); + owner = new OwnedFileHandle(await open(path, "wx+", 0o600)); + const file = owner.handle; + const stage = async (part: string) => { + checkCancellation(); + await file.writeFile(part); + checkCancellation(); + }; + for (const part of buffered) await stage(part); + await stage(firstStagedPart); + for (const part of parts) await stage(part); + + checkCancellation(); + beginOutput(); + // Early iterator return must not invalidate the descriptor behind its + // owner. The lease closes the handle after pending reads settle. + const chunks: AsyncIterable = file + .createReadStream({ + start: 0, + autoClose: false, + encoding: "utf8", + }) + .iterator({ destroyOnReturn: false }); + for await (const chunk of chunks) { + checkCancellation(); + await writeOutputText(destination, chunk); + checkCancellation(); + } + } catch (cause: unknown) { + failure = { cause }; + } + const cleanup = await resources.release({ + kind: "lease", + resource: directory, + lease, + }); + // An interrupt during the last close/removal is still this operation's. + const lateCancellation = cancelled(); + if (failure === undefined && lateCancellation !== undefined) + failure = { cause: lateCancellation }; + if (cleanup.kind === "failed") { + retainedStages.add(resources); + retained = true; + const error = new ToonOutputCleanupFailure( + failure, + operation, + lease.cleanupResources ?? [directory], + cleanup.cause, + ); + new AnalysisCleanupLeaseGroup(error, [lease]); + throw error; + } + if (failure !== undefined) throw failure.cause; + } finally { + cancellation.finish(); + if (!retained) cancellation.close(); + } + } finally { + parts.return(undefined); + } +}; diff --git a/src/cliOneShot.ts b/src/cliOneShot.ts index 47666bb33b..ae45592a11 100644 --- a/src/cliOneShot.ts +++ b/src/cliOneShot.ts @@ -126,7 +126,8 @@ export const runOneShotCli = async ( const { loadCliRuntime } = await import("./cli/cliLoader.js"); const runtime = await loadCliRuntime(global, host.environment); host.ownCliResources(runtime.closeCliResources); - const { createCliResultOutput } = await import("./cli/cliResultOutput.js"); + const { createCliResultOutput, cliOutputCancellationStatus } = + await import("./cli/cliResultOutput.js"); let wroteOutput = false; const resultOutput = createCliResultOutput( outputOptions, @@ -150,6 +151,11 @@ export const runOneShotCli = async ( }, }); } catch (cause) { + const cancelledStatus = cliOutputCancellationStatus(cause); + if (cancelledStatus !== undefined) { + host.exit(cancelledStatus, async () => resultOutput?.finish()); + return; + } if (wroteOutput) throw cause; const recovered = await resultOutput ?.recoverFormattingFailure(cause) diff --git a/tests/acceptance/cliFormattingCapacity.test.ts b/tests/acceptance/cliFormattingCapacity.test.ts index 62f3963420..f420ada935 100644 --- a/tests/acceptance/cliFormattingCapacity.test.ts +++ b/tests/acceptance/cliFormattingCapacity.test.ts @@ -11,25 +11,39 @@ import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js"; // Exercise the real dispatcher and the pinned TOON and YAML encoders with an // authored capacity failure. No engine-sized document or native target // execution is needed. -const encoderFailure = (value: string) => ` +const encoderFailure = (value: string, boundary = "") => ` if (${value}?.operation === "inspect_managed_artifact" || ${value}?.data?.operation === "inspect_managed_artifact" || ${value}?.code === "target_unavailable" || ${value}?.error?.code === "target_unavailable") { - process.stderr.write("REA_AUTHORED_FORMAT_CAPACITY\\n"); + process.stderr.write("REA_AUTHORED_FORMAT_CAPACITY${boundary}\\n"); throw new RangeError("Invalid string length"); } `; +// Default TOON and token presentations have different producing entrypoints. +// Distinct markers prove which boundary each scenario actually exercises. const encoders = [ { file: "/@toon-format/toon/dist/index.mjs", - signature: "function encode(input, options) {", - failure: encoderFailure("input"), + boundaries: [ + { + signature: "function encode(input, options) {", + failure: encoderFailure("input", ":encode"), + }, + { + signature: "function encodeLines(input, options) {", + failure: encoderFailure("input", ":encodeLines"), + }, + ], }, { file: "/yaml/dist/public-api.js", - signature: "function stringify(value, replacer, options) {", - failure: encoderFailure("value"), + boundaries: [ + { + signature: "function stringify(value, replacer, options) {", + failure: encoderFailure("value"), + }, + ], }, ]; const encoderCapacityHook = `data:text/javascript,${encodeURIComponent(` @@ -40,9 +54,12 @@ registerHooks({ const loaded = next(url, context); const encoder = encoders.find(({ file }) => url.endsWith(file)); if (encoder === undefined) return loaded; - const source = String(loaded.source); - if (!source.includes(encoder.signature)) throw new Error("Pinned encoder boundary changed: " + encoder.file); - return { ...loaded, source: source.replace(encoder.signature, encoder.signature + encoder.failure) }; + let source = String(loaded.source); + for (const boundary of encoder.boundaries) { + if (!source.includes(boundary.signature)) throw new Error("Pinned encoder boundary changed: " + encoder.file + " " + boundary.signature); + source = source.replace(boundary.signature, boundary.signature + boundary.failure); + } + return { ...loaded, source }; }, }); `)}`; @@ -69,18 +86,46 @@ type FormattingScenario = { readonly full: boolean; readonly format?: "yaml" | "md"; readonly human?: boolean; + readonly toonEncoder?: "encode" | "encodeLines"; }; const scenarios: readonly FormattingScenario[] = [ - { name: "default TOON", arguments: [], full: false }, - { name: "human TOON", arguments: [], full: false, human: true }, - { name: "full TOON", arguments: ["--full-output"], full: true }, - { name: "TOON token count", arguments: ["--token-count"], full: false }, - { name: "TOON token window", arguments: ["--token-limit", "3"], full: false }, + { + name: "default TOON", + arguments: [], + full: false, + toonEncoder: "encodeLines", + }, + { + name: "human TOON", + arguments: [], + full: false, + human: true, + toonEncoder: "encodeLines", + }, + { + name: "full TOON", + arguments: ["--full-output"], + full: true, + toonEncoder: "encodeLines", + }, + { + name: "TOON token count", + arguments: ["--token-count"], + full: false, + toonEncoder: "encode", + }, + { + name: "TOON token window", + arguments: ["--token-limit", "3"], + full: false, + toonEncoder: "encode", + }, { name: "full TOON token window", arguments: ["--full-output", "--token-limit", "3"], full: true, + toonEncoder: "encode", }, { name: "YAML", format: "yaml", arguments: [], full: false }, { @@ -132,7 +177,9 @@ for (const scenario of scenarios) { cwd: workspace.root, env: { PATH: process.env.PATH ?? "" } }, ); expect(result.exitCode, result.stderr).toBe(1); - expect(result.stderr.trim()).toBe("REA_AUTHORED_FORMAT_CAPACITY"); + expect(result.stderr.trim()).toBe( + `REA_AUTHORED_FORMAT_CAPACITY${scenario.toonEncoder === undefined ? "" : `:${scenario.toonEncoder}`}`, + ); expect(result.stdout).toContain("resource_constraint"); expect(result.stdout).toContain("--format json"); expect(result.stdout).toContain("formatted_characters"); @@ -195,7 +242,7 @@ workspaceCliTest.skipIf(process.platform === "win32")( ); expect({ code, stderr }).toEqual({ code: 141, - stderr: "REA_AUTHORED_FORMAT_CAPACITY\n", + stderr: "REA_AUTHORED_FORMAT_CAPACITY:encodeLines\n", }); }, ); diff --git a/tests/acceptance/cliToonCancellation.test.ts b/tests/acceptance/cliToonCancellation.test.ts new file mode 100644 index 0000000000..acdfeb4ef4 --- /dev/null +++ b/tests/acceptance/cliToonCancellation.test.ts @@ -0,0 +1,138 @@ +import { mkdir, readdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import { expect } from "vitest"; + +import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js"; +import { awaitAcknowledgement } from "../fixtures/lifecycleAcknowledgement.js"; +import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js"; + +// Only the encoder's document size and one admitted I/O completion are +// controlled. The real dispatcher owns signals, stdout and staged resources. +const stagingHook = (phase: "stage" | "delivery" | "cleanup") => + `data:text/javascript,${encodeURIComponent(` +import fs from "node:fs/promises"; +import { registerHooks, syncBuiltinESMExports } from "node:module"; +let held = false; +const gate = async () => { + if (held) return; + held = true; + const released = new Promise(resolve => process.stdin.once("data", resolve)); + process.stderr.write("REA_TOON_PHASE:${phase}\\n"); + await released; + process.stdin.pause(); +}; +const originalOpen = fs.open; +fs.open = async (...args) => { + const file = await originalOpen(...args); + if (String(args[0]).endsWith("/output")) { + const method = ${JSON.stringify(phase === "cleanup" ? "close" : "writeFile")}; + const original = file[method].bind(file); + file[method] = async (...args) => { + if (${JSON.stringify(phase)} !== "delivery") await gate(); + return original(...args); + }; + } + return file; +}; +syncBuiltinESMExports(); +if (${JSON.stringify(phase)} === "delivery") { + const originalWrite = process.stdout.write.bind(process.stdout); + process.stdout.write = function (...args) { + if (!held) { + void gate().then(() => originalWrite(...args)); + return false; + } + return originalWrite(...args); + }; +} +registerHooks({ + load(url, context, next) { + const loaded = next(url, context); + if (!url.endsWith("/@toon-format/toon/dist/index.mjs")) return loaded; + const signature = "function encodeLines(input, options) {"; + const source = String(loaded.source); + if (!source.includes(signature)) throw new Error("Pinned TOON boundary changed"); + return { ...loaded, source: source.replace(signature, signature + + ' if (input?.operation === "inspect_managed_artifact") return Array.from({length: 24}, () => "x".repeat(65536)).values();') }; + } +}); +`)}`; + +workspaceCliTest.skipIf(!["linux", "darwin"].includes(process.platform)).for( + (["stage", "delivery", "cleanup"] as const).flatMap((phase) => + ( + [ + ["SIGINT", 130], + ["SIGTERM", 143], + ] as const + ).map(([signal, status]) => ({ phase, signal, status })), + ), +)( + "TOON $phase retains its stage through $signal and exits $status", + async ({ phase, signal, status }, { processes, workspace }) => { + const target = join(workspace.root, "fixture.dll"); + const temporary = join(workspace.root, "temporary"); + await writeFile(target, buildManagedPeFixture()); + await mkdir(temporary); + let acknowledgePhase!: () => void; + const phaseAcknowledged = new Promise((resolve) => { + acknowledgePhase = resolve; + }); + const child = processes.spawn( + process.execPath, + [ + "--import", + stagingHook(phase), + resolve("scripts/rea.mjs"), + "inspect-managed-artifact", + target, + ], + { + cwd: workspace.root, + env: { TMPDIR: temporary, PATH: process.env.PATH }, + }, + ); + let stderr = ""; + let bytes = 0; + child.stdout.on("data", (chunk: Buffer) => { + bytes += chunk.length; + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + if (stderr.includes(`REA_TOON_PHASE:${phase}\n`)) acknowledgePhase(); + }); + const closed = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal })); + }); + try { + await awaitAcknowledgement( + phaseAcknowledged, + closed, + `TOON ${phase} phase`, + ); + expect(await readdir(temporary)).toHaveLength(1); + expect(child.kill(signal)).toBe(true); + child.stdin.end("release"); + const result = await closed; + expect(await readdir(temporary), JSON.stringify(result)).toEqual([]); + expect(result, stderr).toEqual({ code: status, signal: null }); + expect(stderr.trim()).toBe(`REA_TOON_PHASE:${phase}`); + if (phase === "stage") expect(bytes).toBe(0); + if (phase === "delivery") { + expect(bytes).toBeGreaterThan(0); + expect(bytes).toBeLessThan(24 * (65536 + 1)); + } + // Cleanup starts only after the full controlled document was delivered. + if (phase === "cleanup") expect(bytes).toBe(24 * (65536 + 1)); + } finally { + if (!child.stdin.destroyed) child.stdin.end("release"); + await processes.shutdown(); + await closed; + } + }, +); diff --git a/tests/boundary/cli/androidCliCancellation.test.ts b/tests/boundary/cli/androidCliCancellation.test.ts index 3d18406c0b..6b15330aa5 100644 --- a/tests/boundary/cli/androidCliCancellation.test.ts +++ b/tests/boundary/cli/androidCliCancellation.test.ts @@ -1,6 +1,7 @@ import { EventEmitter } from "node:events"; import { expect, it } from "vitest"; import { + CommandCancellationScope, withCommandCancellation, type CommandCancellationHost, } from "../../../src/cli/commandCancellation.js"; @@ -59,3 +60,26 @@ it("leaves the exit code alone on success and removes listeners when execution t expect(events.listenerCount("SIGINT")).toBe(0); expect(events.listenerCount("SIGTERM")).toBe(0); }); + +it("retains signals after operation settlement until its resource close succeeds", () => { + const events = new EventEmitter(); + const host: CommandCancellationHost = { + on: events.on.bind(events), + off: events.off.bind(events), + exitCode: 0, + }; + const scope = new CommandCancellationScope(host); + scope.finish(); + expect(host.exitCode).toBe(0); + expect(events.listenerCount("SIGINT")).toBe(1); + events.emit("SIGTERM"); + expect(scope.exitStatus).toBe(143); + expect(host.exitCode).toBe(143); + events.emit("SIGINT"); + expect(scope.exitStatus).toBe(143); + expect(scope.signal.reason).toMatchObject({ kind: "terminate" }); + scope.close(); + scope.close(); + expect(events.listenerCount("SIGINT")).toBe(0); + expect(events.listenerCount("SIGTERM")).toBe(0); +}); diff --git a/tests/boundary/filesystem/toonOutput.test.ts b/tests/boundary/filesystem/toonOutput.test.ts new file mode 100644 index 0000000000..3a3de25aea --- /dev/null +++ b/tests/boundary/filesystem/toonOutput.test.ts @@ -0,0 +1,131 @@ +import { constants } from "node:buffer"; +import { createHash } from "node:crypto"; +import { open, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { Writable } from "node:stream"; + +import { decode, encode } from "@toon-format/toon"; +import { expect, it, vi } from "vitest"; + +import { createCliResultOutput } from "../../../src/cli/cliResultOutput.js"; +import { parseCliOutputArguments } from "../../../src/cliOutput.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +vi.mock("node:fs/promises", { spy: true }); + +const metadata = { command: "fixture", duration: "1ms" }; +const outputFor = (destination: Writable) => { + const output = createCliResultOutput( + parseCliOutputArguments([]), + destination, + ); + if (output === undefined) throw new Error("missing output surface"); + return output; +}; + +it("writes staged TOON bytes identical to the ordinary encoder", async () => { + const value = { + rows: Array.from({ length: 800 }, (_, id) => ({ + id, + name: "x".repeat(1600) + ' 🧪, "quote"\nline', + })), + }; + const expected = `${encode(value)}\n`; + const observed = createHash("sha256"); + let writes = 0; + const output = outputFor( + new Writable({ + highWaterMark: 1, + write(part: Buffer, _encoding, callback) { + writes++; + observed.update(part); + setImmediate(callback); + }, + }), + ); + expect(await output.write(value, metadata)).toBe(true); + expect(output.failed).toBe(false); + expect(writes).toBeGreaterThan(1); + expect(observed.digest("hex")).toBe( + createHash("sha256").update(expected).digest("hex"), + ); +}); + +it("reports a late oversized TOON line before any document bytes are written", async () => { + const leaf = "x".repeat(64 * 1024); + const prefix = Array.from({ length: 20 }, () => ({ text: leaf })); + const lateLine = Array.from( + { length: Math.ceil(constants.MAX_STRING_LENGTH / leaf.length) + 1 }, + () => leaf, + ); + const parts: Buffer[] = []; + const output = outputFor( + new Writable({ + write(part: Buffer, _encoding, callback) { + parts.push(part); + callback(); + }, + }), + ); + expect(await output.write({ prefix, lateLine }, metadata)).toBe(true); + const text = Buffer.concat(parts).toString("utf8"); + expect(text).not.toContain("prefix["); + expect(decode(text)).toMatchObject({ + code: "resource_constraint", + remediation: { action: expect.stringContaining("--format json") }, + }); +}); + +it("cleans staged output after a destination failure", async () => { + const cause = new Error("destination closed"); + const path = await createTestTempDirectory("rea-toon-output-"); + const tmp = process.platform === "win32" ? "TEMP" : "TMPDIR"; + const previous = process.env[tmp]; + process.env[tmp] = path; + try { + expect(tmpdir()).toBe(path); + const output = outputFor( + new Writable({ + write(_part, _encoding, callback) { + callback(cause); + }, + }), + ); + const value = { + rows: Array.from({ length: 20 }, () => ({ + value: "x".repeat(64 * 1024), + })), + }; + await expect(output.write(value, metadata)).rejects.toBe(cause); + expect(output.failed).toBe(true); + expect(await readdir(path)).toEqual([]); + } finally { + if (previous === undefined) delete process.env[tmp]; + else process.env[tmp] = previous; + } +}); + +it("returns a typed refusal if temporary storage runs out before stdout starts", async () => { + vi.mocked(open).mockRejectedValueOnce( + Object.assign(new Error("disk full"), { code: "ENOSPC" }), + ); + const chunks: Buffer[] = []; + const output = outputFor( + new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + ); + const leaf = "x".repeat(64 * 1024); + const rows = Array.from({ length: 20 }, () => ({ text: leaf })); + expect(await output.write({ rows }, metadata)).toBe(true); + const text = Buffer.concat(chunks).toString("utf8"); + expect(text).not.toContain("rows["); + expect(decode(text)).toMatchObject({ + code: "resource_constraint", + details: { reported_limits: { system_code: "ENOSPC" } }, + remediation: { action: expect.stringContaining("--format json") }, + }); +}); diff --git a/tests/boundary/filesystem/toonOutputCleanup.test.ts b/tests/boundary/filesystem/toonOutputCleanup.test.ts new file mode 100644 index 0000000000..44c86c4018 --- /dev/null +++ b/tests/boundary/filesystem/toonOutputCleanup.test.ts @@ -0,0 +1,284 @@ +import type { FileHandle } from "node:fs/promises"; +import { open, readdir, rm } from "node:fs/promises"; +import { Writable } from "node:stream"; +import { join } from "node:path"; + +import { decode } from "@toon-format/toon"; +import { expect, it, vi } from "vitest"; + +import { createCliResultOutput } from "../../../src/cli/cliResultOutput.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { closeCliResources } from "../../../src/cli/cliResources.js"; +import { parseCliOutputArguments } from "../../../src/cliOutput.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +vi.mock("node:fs/promises", { spy: true }); +const native = + await vi.importActual("node:fs/promises"); +const value = { + rows: Array.from({ length: 20 }, () => ({ text: "x".repeat(65536) })), +}; +const metadata = { command: "fixture", duration: "1ms" }; + +const withStageFault = async ( + fault: + | "close" + | "remove" + | "capacity-and-close" + | "capacity-and-remove" + | "close-then-remove", + run: (fixture: { + root: string; + events: string[]; + file: () => FileHandle | undefined; + allowCleanup: () => void; + }) => Promise, +) => { + const root = await createTestTempDirectory("rea-toon-cleanup-"); + const variable = process.platform === "win32" ? "TEMP" : "TMPDIR"; + const previous = process.env[variable]; + process.env[variable] = root; + let denied = true; + let recovering = false; + let removalAttempts = 0; + let file: FileHandle | undefined; + let nativeClose: (() => Promise) | undefined; + const events: string[] = []; + vi.mocked(open).mockImplementationOnce(async (...arguments_) => { + file = await native.open(...arguments_); + nativeClose = file.close.bind(file); + let closing = false; + vi.spyOn(file, "close").mockImplementation(async () => { + // FileHandle.close also closes its read stream, which may reenter close. + if (closing) return nativeClose?.(); + events.push("close"); + if ( + (fault === "close" || + fault === "capacity-and-close" || + fault === "close-then-remove") && + denied + ) + throw new Error("close denied"); + closing = true; + try { + await nativeClose?.(); + } finally { + closing = false; + } + }); + if (fault.startsWith("capacity-")) + vi.spyOn(file, "writeFile").mockRejectedValueOnce( + Object.assign(new Error("temporary storage full"), { code: "ENOSPC" }), + ); + return file; + }); + vi.mocked(rm).mockImplementation(async (path, options) => { + if (String(path).startsWith(root)) { + events.push("remove"); + if ( + fault === "close-then-remove" && + !recovering && + !denied && + removalAttempts++ === 0 + ) + throw new Error("remove denied after close recovery"); + if ((fault === "remove" || fault === "capacity-and-remove") && denied) + throw new Error("remove denied"); + } + await native.rm(path, options); + }); + try { + await run({ + root, + events, + file: () => file, + allowCleanup: () => { + denied = false; + }, + }); + } finally { + denied = false; + recovering = true; + try { + await closeCliResources(); + } finally { + if (file !== undefined && file.fd >= 0) await nativeClose?.(); + vi.restoreAllMocks(); + if (previous === undefined) delete process.env[variable]; + else process.env[variable] = previous; + } + } +}; +const outputFor = (parts: Buffer[]) => { + const output = createCliResultOutput( + parseCliOutputArguments([]), + new Writable({ + write(part: Buffer, _encoding, done) { + parts.push(part); + done(); + }, + }), + ); + if (output === undefined) throw new Error("Missing CLI result output"); + return output; +}; + +it("retains a failed descriptor close and its root through shutdown retries", async () => { + await withStageFault("close", async ({ root, events, allowCleanup }) => { + const failure = await outputFor([]) + .write(value, metadata) + .then( + () => { + throw new Error("Expected cleanup failure"); + }, + (cause: unknown) => cause, + ); + expect(await readdir(root)).toHaveLength(1); + expect(events).toEqual(["close"]); + expect(failure).toMatchObject({ cleanupIncomplete: true }); + await expect(closeCliResources()).rejects.toThrow(); + expect(events).toEqual(["close", "close"]); + expect(await readdir(root)).toHaveLength(1); + allowCleanup(); + await closeCliResources(); + expect(events).toEqual(["close", "close", "close", "remove"]); + expect(await readdir(root)).toEqual([]); + expect(failure).toMatchObject({ + cleanupIncomplete: false, + cleanupResources: [], + }); + }); +}); + +it("retries only directory removal after the descriptor has closed", async () => { + await withStageFault( + "remove", + async ({ root, events, file, allowCleanup }) => { + const failure = await outputFor([]) + .write(value, metadata) + .then( + () => { + throw new Error("Expected cleanup failure"); + }, + (cause: unknown) => cause, + ); + expect(file()?.fd).toBe(-1); + expect(await readdir(root)).toHaveLength(1); + await expect(closeCliResources()).rejects.toThrow(); + expect(events).toEqual(["close", "remove", "remove"]); + allowCleanup(); + await closeCliResources(); + expect(events).toEqual(["close", "remove", "remove", "remove"]); + expect(await readdir(root)).toEqual([]); + expect(failure).toMatchObject({ cleanupIncomplete: false }); + }, + ); +}); + +it.each(["capacity-and-close", "capacity-and-remove"] as const)( + "keeps a typed capacity refusal alongside %s", + async (fault) => { + await withStageFault(fault, async ({ root, events, allowCleanup }) => { + const parts: Buffer[] = []; + const output = outputFor(parts); + expect(await output.write(value, metadata)).toBe(true); + const text = Buffer.concat(parts).toString("utf8"); + expect(text).not.toContain("rows["); + expect(decode(text)).toMatchObject({ code: "cleanup_incomplete" }); + expect(text).toContain("resource_constraint"); + expect(text).toContain("ENOSPC"); + expect(text).toContain("--format json"); + expect(events).toEqual( + fault === "capacity-and-close" ? ["close"] : ["close", "remove"], + ); + expect(await readdir(root)).toHaveLength(1); + allowCleanup(); + await closeCliResources(); + expect(await readdir(root)).toEqual([]); + }); + }, +); + +it("retains a failed destination and cleanup without publishing a replacement document", async () => { + await withStageFault("close", async ({ root, allowCleanup }) => { + const cause = new Error("destination closed"); + let writes = 0; + const output = createCliResultOutput( + parseCliOutputArguments([]), + new Writable({ + write(_part, _encoding, done) { + writes++; + done(cause); + }, + }), + ); + if (output === undefined) throw new Error("Missing CLI result output"); + const failure = await output.write(value, metadata).then( + () => { + throw new Error("Expected output and cleanup failure"); + }, + (error: unknown) => error, + ); + expect(output.failed).toBe(true); + expect(writes).toBe(1); + expect(failure).toMatchObject({ + cleanupIncomplete: true, + outputFailure: { cause }, + }); + expect(await readdir(root)).toHaveLength(1); + allowCleanup(); + await closeCliResources(); + expect(await readdir(root)).toEqual([]); + expect(failure).toMatchObject({ cleanupIncomplete: false }); + }); +}); + +it("updates unresolved resources when a recovered close is followed by failed removal", async () => { + await withStageFault( + "close-then-remove", + async ({ root, events, allowCleanup }) => { + const failure = await outputFor([]) + .write(value, metadata) + .then( + () => { + throw new Error("Expected cleanup failure"); + }, + (cause: unknown) => cause, + ); + const [name] = await readdir(root); + const stage = join(root, name!); + expect(failure).toMatchObject({ cleanupIncomplete: true }); + allowCleanup(); + await expect(closeCliResources()).rejects.toThrow(); + expect(failure).toMatchObject({ + cleanupIncomplete: true, + cleanupResources: [stage], + }); + expect(events).toEqual(["close", "close", "remove"]); + await closeCliResources(); + expect(events).toEqual(["close", "close", "remove", "remove"]); + expect(failure).toMatchObject({ + cleanupIncomplete: false, + cleanupResources: [], + }); + }, + ); +}); + +it("refuses staging when the private-root authority cannot establish privacy", async () => { + const refusal = new Error("Private runtime unavailable"); + const create = vi + .spyOn(PrivateRuntimeRoot, "create") + .mockRejectedValueOnce(refusal); + const openStage = vi.mocked(open); + const before = openStage.mock.calls.length; + const parts: Buffer[] = []; + try { + await expect(outputFor(parts).write(value, metadata)).rejects.toBe(refusal); + expect(create).toHaveBeenCalledWith({ prefix: "rea-toon-" }); + expect(openStage.mock.calls.length).toBe(before); + expect(parts).toEqual([]); + } finally { + vi.restoreAllMocks(); + } +});