diff --git a/docs/cli.md b/docs/cli.md index e1575faa0e..f6b7d55b08 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1294,6 +1294,18 @@ new file. A publication that has committed can win a later cancellation; the after commit, the `cleanup_incomplete` error reports the same file as `details.published_path` and `details.bytes`. +When a failure retains a completed observation and no bundle committed, +`details.source_closure` states what that observation cites. `publication` is +`not_published`. `records` holds authentic source Evidence this invocation +captured, within 64 KiB. `omitted` lists up to 64 other identities and +`omitted_count` the total: `not_captured` means the invocation holds no such +record, so nothing is shown for it or its ancestors; `not_inlined` means it was +captured but is not in this response. `complete` is true only when every cited +record is in `records` or embedded in the observation. If examining the closure +itself fails, `collection_failure` names the error type and the closure state is +unknown; the original failure is still returned. Nothing is retained after exit, +and running the command again is a new observation. + When the reader of piped output closes early, as `rea ... | head` does, the CLI stops writing, prints no error and exits `141` (`128 + 13`, the status of a SIGPIPE-terminated writer). The operation itself may have completed; only its diff --git a/src/application/AnalysisEvidenceSources.ts b/src/application/AnalysisEvidenceSources.ts index ff43a6a8b0..b7795ba283 100644 --- a/src/application/AnalysisEvidenceSources.ts +++ b/src/application/AnalysisEvidenceSources.ts @@ -1,6 +1,7 @@ import { readArtifactInspectionSources } from "../domain/artifactInspection.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; -import type { Evidence } from "../domain/evidence.js"; +import { tryParseEvidence, type Evidence } from "../domain/evidence.js"; +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import { err, ok, type Result } from "../domain/result.js"; /** Resolve the embedded publication closure without reinterpreting it in delivery adapters. */ @@ -15,3 +16,128 @@ export const analysisEvidenceSources = ( return err(embedded.error.retainPartialObservation(complete)); return ok([...embedded.value, complete]); }; + +/** Matches the diagnostics allowance retained for other finalized one-shot work. */ +const SOURCE_CLOSURE_INLINE_BYTES = 64 * 1024; +/** Identities beyond this stay counted; direct links remain on the observation. */ +const SOURCE_CLOSURE_LISTED_OMISSIONS = 64; +const SOURCE_CLOSURE_RECOVERY = + "No bundle was written and this invocation retains nothing after exit. Keep any records shown here, or run the operation again after following remediation; that is a new observation and may differ."; + +/** What a failed one-shot delivery holds for one record its observation cites. */ +export type SourceClosureOmission = { + readonly evidence_id: string; + /** + * `not_captured`: this invocation holds no such record, so none can be shown. + * `not_inlined`: captured, but not part of this bounded failure response. + */ + readonly reason: "not_captured" | "not_inlined"; +}; + +/** Bounded failure handoff for a completed observation whose bundle never committed. */ +export type UndeliveredSourceClosure = { + readonly publication: "not_published"; + /** Every cited record is in `records` or embedded in the observation itself. */ + readonly complete: boolean; + readonly records: Evidence[]; + readonly omitted_count: number; + readonly omitted: SourceClosureOmission[]; + readonly recovery: string; +}; + +/** The closure state is unknown because examining it failed. */ +export type UnavailableSourceClosure = { + readonly publication: "not_published"; + readonly complete: false; + readonly collection_failure: string; + readonly recovery: string; +}; + +const collectSourceClosure = ( + observation: Evidence, + captured: readonly Evidence[], + options: { readonly inlineRecords: boolean }, +): UndeliveredSourceClosure | undefined => { + const inspection = analysisEvidenceSources(observation); + const embedded = new Map( + (inspection.ok ? inspection.value : []).map((record) => [ + record.evidence_id, + record, + ]), + ); + const held = new Map(captured.map((record) => [record.evidence_id, record])); + // Set iteration reaches entries added while it runs, which makes this a + // deduplicating breadth-first queue: directly cited records spend the + // allowance first, and each link costs one lookup however often it repeats. + const closure = new Set([observation.evidence_id]); + for (const evidenceId of observation.evidence_links) closure.add(evidenceId); + for (const evidenceId of held.keys()) closure.add(evidenceId); + const records: Evidence[] = []; + const omitted: SourceClosureOmission[] = []; + let omittedCount = 0; + let inlineBytes = options.inlineRecords ? SOURCE_CLOSURE_INLINE_BYTES : 0; + const omit = ( + evidenceId: string, + reason: SourceClosureOmission["reason"], + ) => { + omittedCount += 1; + if (omitted.length < SOURCE_CLOSURE_LISTED_OMISSIONS) + omitted.push({ evidence_id: evidenceId, reason }); + }; + for (const evidenceId of closure) { + if (evidenceId === observation.evidence_id) continue; + const inObservation = embedded.get(evidenceId); + const record = inObservation ?? held.get(evidenceId); + if (record === undefined) { + omit(evidenceId, "not_captured"); + continue; + } + for (const link of record.evidence_links) closure.add(link); + if (inObservation !== undefined) continue; + const bytes = jsonValueByteLength(record, inlineBytes); + if (bytes > inlineBytes) { + omit(evidenceId, "not_inlined"); + continue; + } + records.push(record); + inlineBytes -= bytes; + } + if (closure.size === 1) return undefined; + return { + publication: "not_published", + complete: omittedCount === 0, + records, + omitted_count: omittedCount, + omitted, + recovery: SOURCE_CLOSURE_RECOVERY, + }; +}; + +/** + * Describe the source closure of a completed observation after delivery failed + * before commit. Only already captured records are reported, so an unresolved + * link stays an explicit omission and its own ancestors remain unknown. + * + * This annotates a failure that must reach the caller, so it never throws. If + * examining the closure fails, the state is reported as unknown, naming only + * the bounded error type rather than a message that could quote analyzed data. + */ +export const undeliveredSourceClosure = ( + partialObservation: unknown, + captured: readonly Evidence[], + options: { readonly inlineRecords: boolean }, +): UndeliveredSourceClosure | UnavailableSourceClosure | undefined => { + try { + const observation = tryParseEvidence(partialObservation); + return observation === undefined + ? undefined + : collectSourceClosure(observation, captured, options); + } catch (cause: unknown) { + return { + publication: "not_published", + complete: false, + collection_failure: cause instanceof Error ? cause.name : "unknown", + recovery: SOURCE_CLOSURE_RECOVERY, + }; + } +}; diff --git a/src/application/EvidenceBundleCommands.ts b/src/application/EvidenceBundleCommands.ts index 43b76ceabb..d88e555dd5 100644 --- a/src/application/EvidenceBundleCommands.ts +++ b/src/application/EvidenceBundleCommands.ts @@ -212,6 +212,11 @@ export const publishEvidenceDelivery = async (input: { return ok({ delivered: input.delivered, evidenceBundle: written.value }); }; +/** A delivery failure can follow commit; its receipt then owns the source closure. */ +export const evidenceDeliveryCommitted = (error: AnalysisError): boolean => + error.diagnostics?.evidence_bundle !== undefined || + (error instanceof EvidenceFileError && error.published !== undefined); + /** Publish a completed bundle and retry its exact staged-output cleanup owner. */ const persistEvidenceBundleCommand = async ( bundle: EvidenceBundle, diff --git a/src/cli/analysisEvidenceDelivery.ts b/src/cli/analysisEvidenceDelivery.ts index 0550210b15..1858bd3fa8 100644 --- a/src/cli/analysisEvidenceDelivery.ts +++ b/src/cli/analysisEvidenceDelivery.ts @@ -1,5 +1,8 @@ import { recordCliEvidencePublication } from "./cliEvidencePublication.js"; -import { analysisEvidenceSources } from "../application/AnalysisEvidenceSources.js"; +import { + analysisEvidenceSources, + undeliveredSourceClosure, +} from "../application/AnalysisEvidenceSources.js"; import type { AnalysisViewRequest } from "../domain/analysisView/analysisView.js"; import { z } from "incur"; import { @@ -14,12 +17,18 @@ import { redactExplicitFailure } from "../domain/explicitSensitiveFailure.js"; import { err, ok, type Result } from "../domain/result.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; import { createEvidenceBundle } from "../domain/evidenceBundle.js"; -import { publishEvidenceDelivery } from "../application/EvidenceBundleCommands.js"; +import { + evidenceDeliveryCommitted, + publishEvidenceDelivery, +} from "../application/EvidenceBundleCommands.js"; import { inspectAnalysisViewValidated } from "../application/analysisView/AnalysisViewService.js"; import { localPathStringSchema } from "../domain/localPath.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; -import { projectCliAnalysisResult } from "./projectCliAnalysisResult.js"; +import { + isObject, + projectCliAnalysisResult, +} from "./projectCliAnalysisResult.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import type { ZodSafeParseResult } from "zod"; import { bindCliEvidenceCallerPath } from "./cliEvidenceCallerPath.js"; @@ -169,6 +178,29 @@ export const deliverCliAnalysisEvidence = async ( }); }; +/** + * A requested bundle that never committed cannot silently drop what a completed + * observation cites. Absent delivery or observation leaves the failure untouched. + */ +const withUndeliveredSourceClosure = ( + failure: JsonValue, + input: CliEvidenceDelivery, +): JsonValue => { + if (input.evidenceOutput === undefined || !isObject(failure)) return failure; + const { details } = failure; + if (!isObject(details)) return failure; + const closure = undeliveredSourceClosure( + details.partial_observation, + input.sourceEvidence ?? [], + // Error output is not the caller's selected Evidence sink. Keep records + // out of it when this invocation carries explicit exclusions. + { inlineRecords: (input.sensitiveValues?.length ?? 0) === 0 }, + ); + return closure === undefined + ? failure + : { ...failure, details: { ...details, source_closure: closure } }; +}; + /** Publish already finalized CLI analysis without opening a second provider/session. */ export const publishCliAnalysisEvidence = async ( result: Result, @@ -179,7 +211,9 @@ export const publishCliAnalysisEvidence = async ( projectAnalysisError( redactExplicitFailure(error, input.sensitiveValues ?? []), ); - if (!result.ok) return result.error; + const undelivered = (error: AnalysisError) => + withUndeliveredSourceClosure(projectFailure(error), input); + if (!result.ok) return withUndeliveredSourceClosure(result.error, input); const complete = parseEvidence(result.value); const inputError = cliEvidenceDeliveryInputError(input); if (inputError !== undefined) { @@ -197,7 +231,7 @@ export const publishCliAnalysisEvidence = async ( }) : ok(complete); if (!delivered.ok) - return projectFailure(delivered.error.retainPartialObservation(complete)); + return undelivered(delivered.error.retainPartialObservation(complete)); // This projection answers the same direct invocation as its parent. Later // inspect_analysis_view calls own their own input and do not inherit it. const selected = @@ -205,7 +239,7 @@ export const publishCliAnalysisEvidence = async ( ? withEvidenceCallerInputPath(delivered.value, complete.caller_input_path) : delivered.value; const inspectionSources = analysisEvidenceSources(complete); - if (!inspectionSources.ok) return projectFailure(inspectionSources.error); + if (!inspectionSources.ok) return undelivered(inspectionSources.error); const published = await publishEvidenceDelivery({ bundle: createEvidenceBundle([ ...new Map( @@ -220,7 +254,10 @@ export const publishCliAnalysisEvidence = async ( operation, signal, }); - if (!published.ok) return projectFailure(published.error); + if (!published.ok) + return evidenceDeliveryCommitted(published.error) + ? projectFailure(published.error) + : undelivered(published.error); const receipt = { ...(input.sensitiveValues?.some( (literal) => diff --git a/src/cli/projectCliAnalysisResult.ts b/src/cli/projectCliAnalysisResult.ts index 3b541c7aa8..61bbc03208 100644 --- a/src/cli/projectCliAnalysisResult.ts +++ b/src/cli/projectCliAnalysisResult.ts @@ -16,7 +16,7 @@ export const projectCliAnalysisResult = ( return projectInputFields(result.error, inputBinding); }; -const isObject = ( +export const isObject = ( value: JsonValue | undefined, ): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); diff --git a/tests/boundary/filesystem/undeliveredSourceClosure.test.ts b/tests/boundary/filesystem/undeliveredSourceClosure.test.ts new file mode 100644 index 0000000000..1ce25fdbf2 --- /dev/null +++ b/tests/boundary/filesystem/undeliveredSourceClosure.test.ts @@ -0,0 +1,627 @@ +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import type { UndeliveredSourceClosure } from "../../../src/application/AnalysisEvidenceSources.js"; +import { runDirectAnalysis } from "../../../src/application/DirectAnalysis.js"; +import { + deliverCliAnalysisEvidence, + publishCliAnalysisEvidence, +} from "../../../src/cli/analysisEvidenceDelivery.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, +} from "../../../src/domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { createArtifactInspection } from "../../../src/domain/artifactInspection.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../../src/domain/evidence.js"; +import { jsonByteLength } from "../../../src/domain/jsonSerialization.js"; +import type { JsonValue } from "../../../src/domain/jsonValue.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; +import { err, ok } from "../../../src/domain/result.js"; +import { OwnedStagedOutput } from "../../../src/filesystem/OwnedStagedOutput.js"; +import { inventoryArtifact } from "../../fixtures/artifactInventory.js"; +import { composedWorkflowFixture } from "../../fixtures/composedWorkflowEvidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const provider = { id: "fixture", name: "Fixture", version: "1" }; + +const source = (name: string, links: readonly string[] = []) => + createEvidence(undefined, provider, { + operation: "fixture_source", + parameters: { name }, + result: { name }, + evidenceLinks: [...links], + }); + +const observed = (links: readonly string[]) => + createEvidence(undefined, provider, { + operation: "fixture_observation", + parameters: {}, + result: { completed: true }, + evidenceLinks: [...links], + }); + +const cancelledWith = (observation: unknown): JsonValue => + projectAnalysisError( + new AnalysisCancelledError("fixture_observation", { + partialObservation: observation as Evidence, + }), + "invocation", + ); + +const delivery = (root: string, sources?: readonly Evidence[]) => ({ + providerId: "fixture", + operation: "fixture_observation", + signal: new AbortController().signal, + detail: "complete" as const, + evidenceOutput: join(root, "evidence.json"), + ...(sources === undefined ? {} : { sourceEvidence: sources }), +}); + +const closureOf = (output: unknown) => + (output as { details: { source_closure: UndeliveredSourceClosure } }).details + .source_closure; + +it("reports never-captured links as explicit omissions without fabricating records", async () => { + const root = await createTestTempDirectory("rea-closure-missing-"); + const held = source("held", ["ev_".padEnd(67, "a")]); + const neverCaptured = "ev_".padEnd(67, "b"); + const observation = observed([held.evidence_id, neverCaptured]); + const closure = closureOf( + await publishCliAnalysisEvidence( + err(cancelledWith(observation)), + delivery(root, [held]), + ), + ); + expect(closure.records.map((record) => record.evidence_id)).toEqual([ + held.evidence_id, + ]); + expect(parseEvidence(closure.records[0])).toEqual(held); + // The ancestor of a held record is unresolved here, not absent. + expect(closure.omitted).toEqual([ + { evidence_id: neverCaptured, reason: "not_captured" }, + { evidence_id: "ev_".padEnd(67, "a"), reason: "not_captured" }, + ]); + expect(closure).toMatchObject({ + publication: "not_published", + complete: false, + omitted_count: 2, + }); + expect(await readdir(root)).toEqual([]); +}); + +it("reports an unlinked captured source and stops at nothing linked or captured", async () => { + const root = await createTestTempDirectory("rea-closure-unlinked-"); + const selected = source("selected"); + const observation = observed([]); + expect( + closureOf( + await publishCliAnalysisEvidence( + err(cancelledWith(observation)), + delivery(root, [selected]), + ), + ).records, + ).toEqual([selected]); + + const failure = cancelledWith(observation); + expect( + await publishCliAnalysisEvidence(err(failure), delivery(root, [])), + ).toBe(failure); + // A self-reference is the observation, not one of its sources. + expect( + await publishCliAnalysisEvidence( + err(failure), + delivery(root, [observation]), + ), + ).toBe(failure); +}); + +it("leaves failures untouched unless a completed observation requested delivery", async () => { + const root = await createTestTempDirectory("rea-closure-untouched-"); + const held = source("held"); + const observation = observed([held.evidence_id]); + const requested = delivery(root, [held]); + + // Without a selected output no source closure was requested on success either. + const noOutput = cancelledWith(observation); + expect( + await publishCliAnalysisEvidence(err(noOutput), { + ...requested, + evidenceOutput: undefined, + }), + ).toBe(noOutput); + + const noObservation = projectAnalysisError( + new AnalysisCancelledError("fixture_observation"), + "invocation", + ); + expect(await publishCliAnalysisEvidence(err(noObservation), requested)).toBe( + noObservation, + ); + + const notEvidence = cancelledWith({ kind: "file-snapshot-read" }); + expect(await publishCliAnalysisEvidence(err(notEvidence), requested)).toBe( + notEvidence, + ); + + // A failed authentication cannot select which sources an observation cites. + const altered = cancelledWith({ + ...observation, + evidence_links: [], + normalized_result: { completed: false }, + }); + expect(await publishCliAnalysisEvidence(err(altered), requested)).toBe( + altered, + ); +}); + +it("bounds records and listed omissions while counting everything", async () => { + const root = await createTestTempDirectory("rea-closure-bounds-"); + const held = Array.from({ length: 200 }, (_, index) => + createEvidence(undefined, provider, { + operation: "fixture_source", + parameters: { index }, + result: { text: "x".repeat(4 * 1024) }, + }), + ); + const missing = Array.from({ length: 100 }, (_, index) => + `ev_${index.toString(16).padStart(4, "0")}`.padEnd(67, "c"), + ); + const observation = observed([ + ...held.map((record) => record.evidence_id), + ...missing, + ]); + const closure = closureOf( + await publishCliAnalysisEvidence( + err(cancelledWith(observation)), + delivery(root, held), + ), + ); + expect(closure.complete).toBe(false); + expect(closure.records.length).toBeGreaterThan(0); + expect(closure.records.length).toBeLessThan(held.length); + const expectedOmissions = held.length + missing.length; + expect(closure.omitted_count).toBe( + expectedOmissions - closure.records.length, + ); + expect(closure.omitted).toHaveLength(64); + expect(jsonByteLength(closure)).toBeLessThan(80 * 1024); +}); + +/** Above V8's default call-argument limit of about 125,000 on a forked main thread. */ +const DATA_SIZED_LINKS = 200_000; + +const digestLink = (index: number) => + `ev_${index.toString(16).padStart(64, "0")}`; + +const wideSource = (links: readonly string[]) => + createEvidence(undefined, provider, { + operation: "fixture_source", + parameters: {}, + result: { wide: true }, + evidenceLinks: links, + }); + +it.each(["final-close cancellation", "refused publication"] as const)( + "keeps the original failure when a captured source has a data-sized link set: %s", + async (failure) => { + const root = await createTestTempDirectory("rea-closure-wide-"); + const links = Array.from({ length: DATA_SIZED_LINKS }, (_, index) => + digestLink(index), + ); + const wide = wideSource(links); + const observation = observed([wide.evidence_id]); + const requested = delivery(root, [wide]); + if (failure === "refused publication") + await writeFile(requested.evidenceOutput, "original"); + const output = await publishCliAnalysisEvidence( + failure === "refused publication" + ? ok(observation) + : err(cancelledWith(observation)), + requested, + ); + expect(output).toMatchObject( + failure === "refused publication" + ? { details: { reason: "exists" } } + : { code: "cancelled" }, + ); + const closure = closureOf(output); + expect(closure.records).toEqual([]); + expect(closure.omitted_count).toBe(DATA_SIZED_LINKS + 1); + expect(closure.omitted).toHaveLength(64); + expect(closure.omitted[0]).toEqual({ + evidence_id: wide.evidence_id, + reason: "not_inlined", + }); + expect(jsonByteLength(closure)).toBeLessThan(16 * 1024); + }, + 60_000, +); + +it("reports a data-sized link set cited by the observation itself", async () => { + const root = await createTestTempDirectory("rea-closure-wide-observation-"); + const closure = closureOf( + await publishCliAnalysisEvidence( + err( + cancelledWith( + observed( + Array.from({ length: DATA_SIZED_LINKS }, (_, index) => + digestLink(index), + ), + ), + ), + ), + delivery(root), + ), + ); + expect(closure.omitted_count).toBe(DATA_SIZED_LINKS); + expect(closure.omitted).toHaveLength(64); +}, 60_000); + +it("examines repeated links once across every captured source", async () => { + const root = await createTestTempDirectory("rea-closure-dedupe-"); + const shared = source("shared"); + const repeated = Array.from( + { length: DATA_SIZED_LINKS }, + () => shared.evidence_id, + ); + const first = wideSource(repeated); + const second = createEvidence(undefined, provider, { + operation: "fixture_source", + parameters: { second: true }, + result: { wide: true }, + evidenceLinks: repeated, + }); + const closure = closureOf( + await publishCliAnalysisEvidence( + err(cancelledWith(observed([first.evidence_id, second.evidence_id]))), + delivery(root, [first, second, shared]), + ), + ); + expect(closure.records).toEqual([shared]); + expect(closure.omitted_count).toBe(2); +}, 60_000); + +it.each([ + ["a captured source", "captured"], + ["the completed observation", "observation"], +] as const)( + "keeps the original failure when examining %s throws", + async (_, thrower) => { + const root = await createTestTempDirectory("rea-closure-throws-"); + const held = source("held"); + const failing = (record: Evidence) => + Object.defineProperty({ ...record }, "evidence_links", { + enumerable: true, + get: () => { + throw new RangeError("quoted analyzed-value text"); + }, + }) as Evidence; + const observation = observed([held.evidence_id]); + const projected = cancelledWith(observation) as { + details: Record; + }; + const partial = + thrower === "observation" + ? (failing(observation) as unknown as JsonValue) + : projected.details.partial_observation!; + const failure = { + ...projected, + details: { ...projected.details, partial_observation: partial }, + } as JsonValue; + const output = (await publishCliAnalysisEvidence( + err(failure), + delivery(root, thrower === "captured" ? [failing(held)] : [held]), + )) as { + code: string; + message: string; + details: Record; + }; + expect(output.code).toBe("cancelled"); + expect(output.message).toBe((failure as { message: string }).message); + expect(output.details.partial_observation).toBe(partial); + expect(output.details.operation).toBe("fixture_observation"); + expect(output.details.source_closure).toEqual({ + publication: "not_published", + complete: false, + collection_failure: "RangeError", + recovery: expect.any(String), + }); + expect(JSON.stringify(output.details.source_closure)).not.toContain( + "quoted analyzed-value text", + ); + expect(await readdir(root)).toEqual([]); + }, +); + +it("spends the allowance on smaller later records after a record does not fit", async () => { + const root = await createTestTempDirectory("rea-closure-greedy-"); + const large = createEvidence(undefined, provider, { + operation: "fixture_source", + parameters: {}, + result: { text: "x".repeat(128 * 1024) }, + }); + const small = source("small"); + const closure = closureOf( + await publishCliAnalysisEvidence( + err( + cancelledWith(observed([large.evidence_id, small.evidence_id].sort())), + ), + delivery(root, [large, small]), + ), + ); + expect(closure.records.map((record) => record.evidence_id)).toEqual([ + small.evidence_id, + ]); + expect(closure.omitted).toEqual([ + { evidence_id: large.evidence_id, reason: "not_inlined" }, + ]); +}); + +it("keeps captured records out of error output under explicit exclusions", async () => { + const root = await createTestTempDirectory("rea-closure-sensitive-"); + const held = source("literal-secret-value"); + const output = await publishCliAnalysisEvidence( + err(cancelledWith(observed([held.evidence_id]))), + { ...delivery(root, [held]), sensitiveValues: ["literal-secret-value"] }, + ); + const closure = closureOf(output); + expect(closure.records).toEqual([]); + expect(closure.omitted).toEqual([ + { evidence_id: held.evidence_id, reason: "not_inlined" }, + ]); + expect(JSON.stringify(closure)).not.toContain("literal-secret-value"); +}); + +it("does not duplicate sources an artifact inspection already embeds", async () => { + const root = await createTestTempDirectory("rea-closure-embedded-"); + const input = join(root, "input.txt"); + await writeFile(input, "artifact contents"); + const embedded = createEvidence(undefined, provider, { + operation: "inventory_artifact", + parameters: {}, + result: await inventoryArtifact(input), + }); + const observation = createEvidence(undefined, provider, { + operation: "inspect_artifact", + parameters: {}, + result: createArtifactInspection(embedded), + evidenceLinks: [embedded.evidence_id], + }); + const output = await publishCliAnalysisEvidence( + err(cancelledWith(observation)), + { ...delivery(root), evidenceOutput: join(root, "bundle.json") }, + ); + expect(closureOf(output)).toMatchObject({ + publication: "not_published", + complete: true, + records: [], + omitted_count: 0, + }); +}); + +it("reports the closure when cancellation precedes bundle commit", async () => { + const root = await createTestTempDirectory("rea-closure-precommit-"); + const held = source("held"); + const observation = observed([held.evidence_id]); + const controller = new AbortController(); + controller.abort(new Error("cancelled before publication")); + const output = await publishCliAnalysisEvidence(ok(observation), { + ...delivery(root, [held]), + signal: controller.signal, + }); + expect(output).toMatchObject({ code: "cancelled" }); + expect(closureOf(output).records).toEqual([held]); + expect(await readdir(root)).toEqual([]); +}); + +it("reports the closure when an existing output refuses publication", async () => { + const root = await createTestTempDirectory("rea-closure-exists-"); + const held = source("held"); + const requested = delivery(root, [held]); + await writeFile(requested.evidenceOutput, "original"); + const output = await publishCliAnalysisEvidence( + ok(observed([held.evidence_id])), + requested, + ); + expect(output).toMatchObject({ details: { reason: "exists" } }); + expect(closureOf(output).records).toEqual([held]); + expect(await readFile(requested.evidenceOutput, "utf8")).toBe("original"); +}); + +it("defers to the committed bundle receipt after cancellation wins late", async () => { + const root = await createTestTempDirectory("rea-closure-committed-"); + const held = source("held"); + const controller = new AbortController(); + const close = OwnedStagedOutput.prototype.close; + const seam = vi + .spyOn(OwnedStagedOutput.prototype, "close") + .mockImplementationOnce(async function (this: OwnedStagedOutput) { + await close.call(this); + controller.abort(new Error("cancelled after commit")); + }); + const requested = { + ...delivery(root, [held]), + signal: controller.signal, + }; + let output: JsonValue; + try { + output = await publishCliAnalysisEvidence( + ok(observed([held.evidence_id])), + requested, + ); + } finally { + seam.mockRestore(); + } + expect(output).toMatchObject({ + code: "cancelled", + details: { + diagnostics: { evidence_bundle: { path: requested.evidenceOutput } }, + }, + }); + expect(output).not.toHaveProperty("details.source_closure"); +}); + +it("defers to the published file when staging cleanup stays incomplete after commit", async () => { + const root = await createTestTempDirectory("rea-closure-published-"); + const held = source("held"); + const requested = delivery(root, [held]); + const close = OwnedStagedOutput.prototype.close; + const owners = new Set(); + const seam = vi + .spyOn(OwnedStagedOutput.prototype, "close") + .mockImplementation(async function (this: OwnedStagedOutput) { + owners.add(this); + throw new Error("controlled staging cleanup failure"); + }); + let output: JsonValue; + try { + output = await publishCliAnalysisEvidence( + ok(observed([held.evidence_id])), + requested, + ); + } finally { + seam.mockRestore(); + for (const owner of owners) await close.call(owner); + } + expect(output).toMatchObject({ + code: "cleanup_incomplete", + details: { published_path: requested.evidenceOutput }, + }); + expect(output).not.toHaveProperty("details.source_closure"); +}); + +it("does not change ordinary successful delivery", async () => { + const root = await createTestTempDirectory("rea-closure-success-"); + const held = source("held"); + const observation = observed([held.evidence_id]); + const requested = delivery(root, [held]); + const delivered = await publishCliAnalysisEvidence( + ok(observation), + requested, + ); + expect(JSON.stringify(delivered)).toBe(JSON.stringify(observation)); + const withoutOutput = await publishCliAnalysisEvidence(ok(observation), { + ...requested, + evidenceOutput: undefined, + }); + expect(JSON.stringify(withoutOutput)).toBe(JSON.stringify(observation)); +}); + +it.each(["interrupt", "cleanup-failure", "interrupt-cleanup-failure"] as const)( + "states the sources a production direct analysis never captured: %s", + async (scenario) => { + const workflow = await composedWorkflowFixture(); + const controller = new AbortController(); + const close = workflow.session.close.bind(workflow.session); + workflow.session.close = async () => { + // The fixture's own final close retries this owner after the command. + workflow.session.close = close; + if (scenario !== "cleanup-failure") controller.abort(); + const closed = await close(); + if (scenario === "interrupt" || !closed.ok) return closed; + return err( + new ProviderCleanupError( + "fixture", + ["owned-root"], + { reason: "controlled session close failure" }, + { operation: "close_binary" }, + ), + ); + }; + const root = await createTestTempDirectory("rea-closure-direct-"); + const complete = await runDirectAnalysis( + workflow.dependencies, + workflow.path, + "binary_overview", + {}, + { signal: controller.signal }, + ); + const output = await publishCliAnalysisEvidence(complete, { + ...delivery(root), + signal: controller.signal, + }); + const failure = output as { + code: string; + details: { partial_observation: unknown }; + }; + expect(failure.code).toBe( + scenario === "interrupt" ? "cancelled" : "cleanup_incomplete", + ); + const links = parseEvidence( + failure.details.partial_observation, + ).evidence_links; + expect(links.length).toBeGreaterThan(0); + expect(closureOf(output)).toMatchObject({ + publication: "not_published", + complete: false, + records: [], + omitted_count: links.length, + omitted: links.map((evidence_id) => ({ + evidence_id, + reason: "not_captured", + })), + }); + expect(await readdir(root)).toEqual([]); + }, +); + +it("reports through the shared one-shot delivery when cancellation arrives during close", async () => { + const root = await createTestTempDirectory("rea-closure-deliver-"); + const held = source("held"); + const uncaptured = "ev_".padEnd(67, "d"); + const observation = observed([held.evidence_id, uncaptured]); + const controller = new AbortController(); + const output = await deliverCliAnalysisEvidence({ + ...delivery(root, [held]), + signal: controller.signal, + run: async () => ok(observation), + service: { + close: async () => { + controller.abort(new Error("cancelled during final close")); + }, + }, + }); + expect(output).toMatchObject({ + code: "cancelled", + details: { partial_observation: { evidence_id: observation.evidence_id } }, + }); + expect(closureOf(output)).toMatchObject({ + complete: false, + records: [held], + omitted: [{ evidence_id: uncaptured, reason: "not_captured" }], + }); + expect(await readdir(root)).toEqual([]); +}); + +it("reports the closure when delivered view projection refuses a completed observation", async () => { + const root = await createTestTempDirectory("rea-closure-view-"); + const held = source("held"); + const observation = observed([held.evidence_id]); + const output = await publishCliAnalysisEvidence(ok(observation), { + ...delivery(root, [held]), + detail: "summary", + // A view applying to another provider's facts cannot be selected here. + view: { kind: "native", facet: "pseudocode", offset: 0, limit: 1 }, + }); + expect(closureOf(output).records).toEqual([held]); + expect(await readdir(root)).toEqual([]); +}); + +it("leaves a primary failure without a completed observation untouched", async () => { + const root = await createTestTempDirectory("rea-closure-primary-"); + const held = source("held"); + const primary = projectAnalysisError( + new AnalysisOutputError("fixture_observation", "primary reply failure"), + "invocation", + ); + expect( + await publishCliAnalysisEvidence(err(primary), delivery(root, [held])), + ).toBe(primary); +}); diff --git a/tests/process-global/oneShotSourceClosure.test.ts b/tests/process-global/oneShotSourceClosure.test.ts new file mode 100644 index 0000000000..2ce020b86c --- /dev/null +++ b/tests/process-global/oneShotSourceClosure.test.ts @@ -0,0 +1,379 @@ +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Cli } from "incur"; +import { afterEach, expect, it, vi } from "vitest"; + +import type { UndeliveredSourceClosure } from "../../src/application/AnalysisEvidenceSources.js"; +import * as matchingService from "../../src/application/native/NativeFunctionMatchingService.js"; +import { ArtifactResourceScope } from "../../src/artifacts/ArtifactResourceScope.js"; +import { registerNativeFunctionMatchingCommands } from "../../src/cli/nativeFunctionMatchingCommands.js"; +import { analysisErrorProjectionSchema } from "../../src/contracts/errorSchemas.js"; +import { createEvidence, parseEvidence } from "../../src/domain/evidence.js"; +import { + createEvidenceBundle, + parseEvidenceBundle, +} from "../../src/domain/evidenceBundle.js"; +import { jsonByteLength } from "../../src/domain/jsonSerialization.js"; +import { ProviderCleanupError } from "../../src/domain/providerCleanupError.js"; +import { silentLogger } from "../../src/logger.js"; +import { + nativeMatchingInventory, + nativeMatchingProvider, +} from "../fixtures/nativeFunctionMatching.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +const host = vi.hoisted(() => ({ + listeners: new Map void>(), + get exitCode() { + return process.exitCode; + }, + set exitCode(value: typeof process.exitCode) { + process.exitCode = value; + }, + on(signal: string, listener: () => void) { + this.listeners.set(signal, listener); + }, + off(signal: string, listener: () => void) { + if (this.listeners.get(signal) === listener) this.listeners.delete(signal); + }, +})); + +vi.mock("../../src/cli/commandCancellation.js", async (importOriginal) => { + const original = + await importOriginal< + typeof import("../../src/cli/commandCancellation.js") + >(); + return { + ...original, + withCommandCancellation: ( + operation: (signal: AbortSignal) => Promise, + ) => original.withCommandCancellation(operation, host), + }; +}); + +afterEach(() => { + vi.restoreAllMocks(); + host.listeners.clear(); +}); + +type FinalClose = + | "control" + | "interrupt" + | "cleanup-failure" + | "interrupt-cleanup-failure"; + +const fixture = async (dependencyText = "Synthetic dependency") => { + const root = await createTestTempDirectory("rea-source-closure-"); + const dependency = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_dependency", + parameters: {}, + result: { source: dependencyText }, + }); + const left = nativeMatchingInventory( + "1", + [{ address: "0x10", value: "main" }], + [dependency.evidence_id], + ); + const right = nativeMatchingInventory("2", [ + { address: "0x20", value: "main" }, + ]); + const sources = join(root, "sources.json"); + await writeFile(sources, JSON.stringify(createEvidenceBundle([dependency]))); + const request = join(root, "request.json"); + await writeFile(request, JSON.stringify({ left, right, detail: "complete" })); + return { + root, + dependency, + left, + right, + request, + sources, + output: join(root, "evidence.json"), + }; +}; + +/** + * Run the registered command, faulting only the close of the scope the command + * gave to matching, after that matching completed. Another scope's close cannot + * move the fault, and a command that stops reaching this close fails here. + */ +const runMatching = async ( + input: Awaited>, + finalClose: FinalClose, + options: { + readonly environment?: Readonly>; + readonly earlierScopeClose?: boolean; + } = {}, +) => { + const match = matchingService.matchNativeFunctionsFromBundles; + const close = ArtifactResourceScope.prototype.close; + let commandScope: ArtifactResourceScope | undefined; + let completed = false; + let finalCloses = 0; + const matching = vi + .spyOn(matchingService, "matchNativeFunctionsFromBundles") + .mockImplementation(async (request, paths, resources, matchOptions) => { + commandScope = resources; + if (options.earlierScopeClose === true) + await new ArtifactResourceScope().close(); + const result = await match(request, paths, resources, matchOptions); + completed = result.ok; + return result; + }); + const seam = vi + .spyOn(ArtifactResourceScope.prototype, "close") + .mockImplementation(async function (this: ArtifactResourceScope) { + if (this !== commandScope || !completed) return close.call(this); + finalCloses += 1; + if ( + finalClose === "interrupt" || + finalClose === "interrupt-cleanup-failure" + ) + host.listeners.get("SIGINT")?.(); + await close.call(this); + if ( + finalClose === "cleanup-failure" || + finalClose === "interrupt-cleanup-failure" + ) + throw new ProviderCleanupError( + "fixture", + ["owned-root"], + { reason: "controlled final close failure" }, + { operation: "match_native_functions" }, + ); + }); + const cli = Cli.create("one-shot-source-closure-test", { sync: false }); + registerNativeFunctionMatchingCommands( + cli, + silentLogger, + options.environment ?? {}, + ); + let stdout = ""; + const previous = process.exitCode; + try { + process.exitCode = undefined; + await cli.serve( + [ + "match-native-functions", + input.request, + "--source_bundle", + input.sources, + "--evidence_output", + input.output, + "--json", + ], + { + stdout: (part) => { + stdout += part; + }, + exit: () => {}, + }, + ); + expect(finalCloses, stdout).toBe(1); + return { stdout, exitCode: process.exitCode }; + } finally { + process.exitCode = previous; + seam.mockRestore(); + matching.mockRestore(); + } +}; + +/** Above V8's default call-argument limit of about 125,000 on a forked main thread. */ +const DATA_SIZED_LINKS = 200_000; + +const closureOf = (stdout: string) => { + const failure = analysisErrorProjectionSchema.parse(JSON.parse(stdout)); + const closure = failure.details?.source_closure as unknown as + | UndeliveredSourceClosure + | undefined; + if (closure === undefined) throw new Error(`No source closure: ${stdout}`); + return { failure, closure }; +}; + +const recordIds = (closure: UndeliveredSourceClosure) => + closure.records.map((record) => parseEvidence(record).evidence_id).sort(); + +it.each([ + ["interrupt", "cancelled", 130], + ["cleanup-failure", "cleanup_incomplete", 1], + ["interrupt-cleanup-failure", "cleanup_incomplete", 130], +] as const)( + "keeps captured linked sources through final close: %s", + async (finalClose, code, exitCode) => { + const input = await fixture(); + const { stdout, exitCode: observedExit } = await runMatching( + input, + finalClose, + ); + expect(observedExit).toBe(exitCode); + const { failure, closure } = closureOf(stdout); + expect(failure.code).toBe(code); + const observation = parseEvidence(failure.details?.partial_observation); + expect(observation.evidence_links).toEqual( + [input.left.evidence_id, input.right.evidence_id].sort(), + ); + // Sources are authentic originals, including the transitively linked one. + expect(recordIds(closure)).toEqual( + [ + input.dependency.evidence_id, + input.left.evidence_id, + input.right.evidence_id, + ].sort(), + ); + expect(closure).toMatchObject({ + publication: "not_published", + complete: true, + omitted_count: 0, + omitted: [], + }); + // Late cancellation never turns into successful or silent publication. + expect(await readdir(input.root)).not.toContain("evidence.json"); + expect(host.listeners.size).toBe(0); + }, +); + +it("keeps the fault on the final close when another scope closes first", async () => { + const input = await fixture(); + const { stdout, exitCode } = await runMatching(input, "interrupt", { + earlierScopeClose: true, + }); + expect(exitCode).toBe(130); + const { failure, closure } = closureOf(stdout); + expect(failure.code).toBe("cancelled"); + expect(parseEvidence(failure.details?.partial_observation).operation).toBe( + "match_native_functions", + ); + expect(closure.complete).toBe(true); +}); + +it.each([ + ["interrupt", "cancelled"], + ["cleanup-failure", "cleanup_incomplete"], +] as const)( + "keeps the typed failure for a captured source with a data-sized link set: %s", + async (finalClose, code) => { + // Repeated links are valid Evidence, and only one distinct record backs them. + const ancestor = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_ancestor", + parameters: {}, + result: { source: "Synthetic ancestor" }, + }); + const wide = createEvidence(undefined, nativeMatchingProvider, { + operation: "fixture_dependency", + parameters: {}, + result: { source: "Synthetic wide dependency" }, + evidenceLinks: Array.from( + { length: DATA_SIZED_LINKS }, + () => ancestor.evidence_id, + ), + }); + const input = await fixture(); + const left = nativeMatchingInventory( + "1", + [{ address: "0x10", value: "main" }], + [wide.evidence_id], + ); + await writeFile( + input.sources, + JSON.stringify(createEvidenceBundle([ancestor, wide])), + ); + await writeFile( + input.request, + JSON.stringify({ left, right: input.right, detail: "complete" }), + ); + const { stdout } = await runMatching(input, finalClose, { + environment: { + REA_NATIVE_MATCHING_REPRESENTATION_BYTES: String(1024 ** 3), + }, + }); + const { failure, closure } = closureOf(stdout); + expect(failure.code).toBe(code); + expect(parseEvidence(failure.details?.partial_observation).operation).toBe( + "match_native_functions", + ); + // The closure is deduplicated: each record is examined and reported once. + expect(recordIds(closure)).toEqual( + [ancestor.evidence_id, left.evidence_id, input.right.evidence_id].sort(), + ); + expect(closure).toMatchObject({ + complete: false, + omitted_count: 1, + omitted: [{ evidence_id: wide.evidence_id, reason: "not_inlined" }], + }); + }, + 60_000, +); + +it("hands off the same closure for summary delivery and permits a clean retry", async () => { + const input = await fixture(); + await writeFile( + input.request, + JSON.stringify({ + left: input.left, + right: input.right, + detail: "summary", + }), + ); + const cancelled = await runMatching(input, "interrupt"); + const { closure } = closureOf(cancelled.stdout); + expect(closure.complete).toBe(true); + + const retried = await runMatching(input, "control"); + expect(retried.exitCode).toBeUndefined(); + expect(JSON.parse(retried.stdout)).not.toHaveProperty( + "details.source_closure", + ); + const saved = parseEvidenceBundle( + JSON.parse(await readFile(input.output, "utf8")), + ); + const savedIds = saved.records.map((record) => record.evidence_id); + expect(savedIds).toEqual(expect.arrayContaining(recordIds(closure))); +}); + +it("keeps an oversized source explicit instead of expanding the failure", async () => { + const input = await fixture("x".repeat(128 * 1024)); + const { stdout } = await runMatching(input, "interrupt"); + const { failure, closure } = closureOf(stdout); + expect(failure.code).toBe("cancelled"); + expect(recordIds(closure)).toEqual( + [input.left.evidence_id, input.right.evidence_id].sort(), + ); + expect(closure).toMatchObject({ + complete: false, + omitted_count: 1, + omitted: [ + { evidence_id: input.dependency.evidence_id, reason: "not_inlined" }, + ], + }); + expect(jsonByteLength(closure)).toBeLessThan(64 * 1024); + + // The omitted record stays reachable through ordinary delivery. + const retried = await runMatching(input, "control"); + expect(retried.exitCode).toBeUndefined(); + const saved = parseEvidenceBundle( + JSON.parse(await readFile(input.output, "utf8")), + ); + expect(saved.records.map((record) => record.evidence_id)).toContain( + input.dependency.evidence_id, + ); +}); + +it("leaves ordinary successful delivery free of closure reporting", async () => { + const input = await fixture(); + const { stdout, exitCode } = await runMatching(input, "control"); + expect(exitCode).toBeUndefined(); + expect(stdout).not.toContain("source_closure"); + const delivered = parseEvidence(JSON.parse(stdout)); + const saved = parseEvidenceBundle( + JSON.parse(await readFile(input.output, "utf8")), + ); + expect(saved.records.map((record) => record.evidence_id).sort()).toEqual( + [ + delivered.evidence_id, + input.dependency.evidence_id, + input.left.evidence_id, + input.right.evidence_id, + ].sort(), + ); +});