diff --git a/docs/guides/runtime/process-capture.md b/docs/guides/runtime/process-capture.md index 3f2f0b7207..56c3ab00ff 100644 --- a/docs/guides/runtime/process-capture.md +++ b/docs/guides/runtime/process-capture.md @@ -639,6 +639,31 @@ current token proof is still required before signaling. Swept cleanup verifies t remaining family after its group signals. Coverage is bounded by the native snapshots and generations inspected; sampled group IDs select candidates. +Settlement records the observation before final owned-process cleanup. Even a +quiesced sampled group can have token-owned descendants discovered during that +cleanup. If the ownership layer reports that cleanup sent termination signals, +`settlement.cleanup_outcome` is `cleaned` on success, and process residual +unknowns explain that the intervention may have ended further terminal or +filesystem work. Failed resource cleanup remains `failed` and retains the same +intervention fact in partial observations. This does not establish how an +uncaptured run would have continued or change which processes REA may signal. + +An observed signal is retained even if a later group signal or final ownership +verification fails. It does not establish complete cleanup: the failed resource, +original diagnostic, and retry authority remain visible. A successful retry can +clear the live cleanup status without rewriting the earlier partial observations. + +If capture completion fails before a capture exists, the intervention is retained +in `partial_observation.observations.residual_unknowns`, alongside the available +observations and explicit unavailable fields. Older records may omit this optional +array. No intervention entry is added without an observed signaling fact; absence +does not prove that cleanup sent no signals. + +Readers that previously accepted only `not_required` for a quiesced settlement +must refresh the advertised MCP schema or update their saved-capture reader to +accept `cleaned` as well. Existing saved captures remain valid and keep their +recorded outcome; missing historical signal facts are not reconstructed. + When the host withholds an unrelated process's ownership token, REA leaves that process untouched and records its PID and reason in `cleanup.unverified_processes` and process residual unknowns. Successful cleanup verifies the owned group; diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.ts index 642ec3a006..4d7b1478a5 100644 --- a/src/domain/process/processCapture.ts +++ b/src/domain/process/processCapture.ts @@ -191,7 +191,7 @@ export type RecordProcessCaptureEvent = ( index: number, ) => void; -/** Observed process-tree settlement and the cleanup required by that state. */ +/** Observed process-tree settlement and subsequent owned-resource cleanup. */ export type ProcessSettlement = | VerifiedProcessSettlement | { @@ -204,7 +204,7 @@ export type VerifiedProcessSettlement = | { readonly state: "quiesced"; readonly elapsed_ms: number; - readonly cleanup_outcome: "not_required"; + readonly cleanup_outcome: "not_required" | "cleaned"; } | { readonly state: "alive_at_deadline" | "unverifiable"; @@ -353,6 +353,10 @@ export interface IncompleteProcessCaptureObservations { readonly manifest: PartialProcessObservationField< UnverifiedProcessCapture["manifest"] >; + /** Facts retained after completion failed; historical records may omit them. */ + readonly residual_unknowns?: + | UnverifiedProcessCapture["residual_unknowns"] + | undefined; } const fileStateSchema = z.discriminatedUnion("type", [ @@ -429,7 +433,7 @@ const processSettlementSchema = z.discriminatedUnion("state", [ z.object({ state: z.literal("quiesced"), elapsed_ms: z.int().nonnegative(), - cleanup_outcome: z.enum(["not_required", "failed"]), + cleanup_outcome: z.enum(["not_required", "cleaned", "failed"]), }), z.object({ state: z.enum(["alive_at_deadline", "unverifiable"]), @@ -540,7 +544,7 @@ const processCaptureShapeSchema = z.strictObject({ z.object({ state: z.literal("quiesced"), elapsed_ms: z.int().nonnegative(), - cleanup_outcome: z.literal("not_required"), + cleanup_outcome: z.enum(["not_required", "cleaned"]), }), z.object({ state: z.enum(["alive_at_deadline", "unverifiable"]), @@ -655,6 +659,8 @@ const incompleteProcessCaptureObservationsSchema = z.strictObject({ manifest: partialObservationFieldSchema( processCaptureShapeSchema.shape.manifest, ), + residual_unknowns: + processCaptureShapeSchema.shape.residual_unknowns.optional(), }); const partialCleanupReportSchema = z.strictObject({ @@ -799,7 +805,10 @@ export const partialProcessCaptureObservationSchema = z capture.settlement.state === "quiesced" ? { ...capture.settlement, - cleanup_outcome: "not_required" as const, + cleanup_outcome: + capture.settlement.cleanup_outcome === "cleaned" + ? ("cleaned" as const) + : ("not_required" as const), } : { ...capture.settlement, cleanup_outcome: "cleaned" as const }; const validationCapture: UnverifiedProcessCapture = { diff --git a/src/domain/process/processCapture.validation.test.ts b/src/domain/process/processCapture.validation.test.ts index 659a4878d5..5a21323b1e 100644 --- a/src/domain/process/processCapture.validation.test.ts +++ b/src/domain/process/processCapture.validation.test.ts @@ -514,9 +514,9 @@ it("does not infer normalized retention budget from missing scenario defaults", ).toContain("manifest.scenario.limits.output_bytes"); }); -it("rejects settlement and cleanup combinations that cannot occur", () => { +it("accepts post-quiescence cleanup and rejects unverified settlement combinations", () => { const capture = emptyCapture(); - expect(() => + expect( parseProcessCapture({ ...capture, settlement: { @@ -525,7 +525,9 @@ it("rejects settlement and cleanup combinations that cannot occur", () => { cleanup_outcome: "cleaned", }, }), - ).toThrow("cleanup_outcome"); + ).toMatchObject({ + settlement: { state: "quiesced", cleanup_outcome: "cleaned" }, + }); expect(() => parseProcessCapture({ ...capture, diff --git a/src/process/ProcessOwnership.test.ts b/src/process/ProcessOwnership.test.ts index f150169140..ec2d536849 100644 --- a/src/process/ProcessOwnership.test.ts +++ b/src/process/ProcessOwnership.test.ts @@ -1,10 +1,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { + cleanupFailureFollowedSignal, cleanupOwnedProcessGroup, type ProcessIdentityObservation, type ProcessOwnershipHost, type ProcessTableEntry, } from "./ProcessOwnership.js"; +import { ProcessOwnershipInspectionCancelled } from "./ProcessInspectionCancellation.js"; import { parseProcessEnvironment } from "./ProcessOwnershipObservation.js"; import { selectCapturedProcessGroupIds } from "./ProcessOwnershipProcessTree.js"; import { matchesOwnedProcessCommand } from "./ProcessCommandIdentity.js"; @@ -439,6 +441,7 @@ describe("withheld process evidence during cleanup", () => { expect(result.reason).toBe( "owned process group signal failed: signal denied", ); + if (mode === "signal-error") expect(result.signaled).not.toBe(true); }, ); @@ -611,6 +614,96 @@ describe("sampled detached process groups during cleanup", () => { }); describe("rooted process-group cleanup", () => { + it("retains an earlier signal when a later owned group cannot be signaled", async () => { + const processes = [ + { pid: 100, parentPid: 1, processGroupId: 100 }, + { pid: 101, parentPid: 100, processGroupId: 101 }, + { pid: 900, parentPid: 1, processGroupId: 900 }, + ].map((entry) => ({ ...entry, state: "S", command: "fixture" })); + const signalGroup = vi.fn((group: number) => { + if (group === 101) + throw Object.assign(new Error("second group denied"), { + code: "EACCES", + }); + }); + const result = await cleanupOwnedProcessGroup(ownership, { + listProcesses: async () => processes, + environment: async (pid) => ({ + REA_PROCESS_RUN_ID: pid === 900 ? "foreign" : "run-token", + }), + signalGroup, + }); + expect(signalGroup.mock.calls).toEqual([ + [100, "SIGKILL"], + [101, "SIGKILL"], + ]); + expect(result).toMatchObject({ cleaned: false, signaled: true }); + if (result.cleaned) throw new Error("Expected failed cleanup"); + expect(result.reason).toContain("second group denied"); + }); + + it("retains a signal when final ownership verification becomes unavailable", async () => { + const fixture = host({ 100: { REA_PROCESS_RUN_ID: "run-token" } }); + let sent = false; + const result = await cleanupOwnedProcessGroup( + { ...ownership, sweepTokenOwnedProcesses: true, captureBaseline: [] }, + { + ...fixture.adapter, + listProcesses: async () => { + if (sent) throw new Error("post-signal inspection unavailable"); + return fixture.adapter.listProcesses(); + }, + signalGroup: (group, signal) => { + fixture.adapter.signalGroup(group, signal); + sent = true; + }, + }, + ); + expect(fixture.signalGroup.mock.calls).toEqual([[100, "SIGKILL"]]); + expect(result).toMatchObject({ cleaned: false, signaled: true }); + if (result.cleaned) throw new Error("Expected failed verification"); + expect(result.reason).toContain("post-signal inspection unavailable"); + }); + + it("marks an inspection failure thrown after a signal so callers keep the intervention", async () => { + const fixture = host({ 100: { REA_PROCESS_RUN_ID: "run-token" } }); + let sent = false; + const cancelled = new ProcessOwnershipInspectionCancelled("stopped"); + const attempt = cleanupOwnedProcessGroup( + { ...ownership, sweepTokenOwnedProcesses: true, captureBaseline: [] }, + { + ...fixture.adapter, + listProcesses: async () => { + if (sent) throw cancelled; + return fixture.adapter.listProcesses(); + }, + signalGroup: (group, signal) => { + fixture.adapter.signalGroup(group, signal); + sent = true; + }, + }, + ); + await expect(attempt).rejects.toBe(cancelled); + expect(fixture.signalGroup.mock.calls).toEqual([[100, "SIGKILL"]]); + expect(cleanupFailureFollowedSignal(cancelled)).toBe(true); + }); + + it("does not mark an inspection failure thrown before any signal", async () => { + const cancelled = new ProcessOwnershipInspectionCancelled("stopped"); + const signalGroup = vi.fn(); + await expect( + cleanupOwnedProcessGroup(ownership, { + listProcesses: async () => { + throw cancelled; + }, + environment: async () => ({}), + signalGroup, + }), + ).rejects.toBe(cancelled); + expect(signalGroup).not.toHaveBeenCalled(); + expect(cleanupFailureFollowedSignal(cancelled)).toBe(false); + }); + it("validates and signals rooted descendant groups once, root first", async () => { const processes = [ { pid: 100, parentPid: 1, processGroupId: 100 }, diff --git a/src/process/ProcessOwnership.ts b/src/process/ProcessOwnership.ts index 7787c1b873..21c853ae88 100644 --- a/src/process/ProcessOwnership.ts +++ b/src/process/ProcessOwnership.ts @@ -286,6 +286,8 @@ export type ProcessCleanupResult = export interface ProcessCleanupFailure { readonly cleaned: false; readonly reason: string; + /** Observed signals survive a later failure; absence does not prove none. */ + readonly signaled?: boolean; readonly failures?: readonly ProcessOwnershipValidationFailure[]; readonly unverified?: readonly OwnershipSweepUnverifiedProcess[]; } @@ -382,9 +384,32 @@ const implicitEvidenceOwners = new WeakMap< ProcessOwnershipEvidenceOwner >(); +// Failures thrown after this module signaled an owned group, so callers can +// keep the observed intervention without changing the failure's identity. +const failuresAfterSignal = new WeakSet(); + +/** Whether a thrown cleanup failure followed an observed owned-group signal. */ +export const cleanupFailureFollowedSignal = (cause: unknown): boolean => + typeof cause === "object" && cause !== null && failuresAfterSignal.has(cause); + export const cleanupOwnedProcessGroup = async ( ownership: OwnedProcessGroup, host: ProcessOwnershipHost = systemHost, +): Promise => { + const attempt = { signaled: false }; + try { + return await cleanupOwnedProcessGroupAttempt(ownership, host, attempt); + } catch (cause: unknown) { + if (attempt.signaled && typeof cause === "object" && cause !== null) + failuresAfterSignal.add(cause); + throw cause; + } +}; + +const cleanupOwnedProcessGroupAttempt = async ( + ownership: OwnedProcessGroup, + host: ProcessOwnershipHost, + attempt: { signaled: boolean }, ): Promise => { if (ownership.evidenceOwner === undefined) { let evidenceOwner = implicitEvidenceOwners.get(ownership); @@ -408,7 +433,15 @@ export const cleanupOwnedProcessGroup = async ( exitSettlement, ); if ("cleaned" in plan) return plan; - let signaled = false; + // A later verification or signal failure cannot erase earlier intervention. + const finish = ( + result: ProcessCleanupResult, + unverified: readonly OwnershipSweepUnverifiedProcess[] = plan.unverified, + ): ProcessCleanupResult => + mergeUnverifiedObservations( + attempt.signaled ? { ...result, signaled: true } : result, + unverified, + ); const pendingGroups = [...plan.signalOrder]; const pending = new Set(pendingGroups); const passes = new Map(); @@ -418,13 +451,10 @@ export const cleanupOwnedProcessGroup = async ( pending.delete(processGroupId); const seen = passes.get(processGroupId) ?? 0; if (seen >= OWNED_GROUP_REVALIDATION_LIMIT) - return mergeUnverifiedObservations( - { - cleaned: false, - reason: `owned process group ${String(processGroupId)} reappeared after ${String(OWNED_GROUP_REVALIDATION_LIMIT)} revalidation passes. This release has no runtime override.`, - }, - plan.unverified, - ); + return finish({ + cleaned: false, + reason: `owned process group ${String(processGroupId)} reappeared after ${String(OWNED_GROUP_REVALIDATION_LIMIT)} revalidation passes. This release has no runtime override.`, + }); passes.set(processGroupId, seen + 1); const revalidation = await revalidateOwnedProcessGroup( processGroupId, @@ -433,40 +463,33 @@ export const cleanupOwnedProcessGroup = async ( plan.tokenOwnedIdentities, exitSettlement, ); - if ("cleaned" in revalidation) - return mergeUnverifiedObservations(revalidation, plan.unverified); + if ("cleaned" in revalidation) return finish(revalidation); for (const groupId of revalidation.escapedProcessGroupIds) { if (groupId === processGroupId || pending.has(groupId)) continue; if ((passes.get(groupId) ?? 0) >= OWNED_GROUP_REVALIDATION_LIMIT) - return mergeUnverifiedObservations( - { - cleaned: false, - reason: `owned process group ${String(groupId)} reappeared after ${String(OWNED_GROUP_REVALIDATION_LIMIT)} revalidation passes. This release has no runtime override.`, - }, - plan.unverified, - ); + return finish({ + cleaned: false, + reason: `owned process group ${String(groupId)} reappeared after ${String(OWNED_GROUP_REVALIDATION_LIMIT)} revalidation passes. This release has no runtime override.`, + }); pending.add(groupId); pendingGroups.push(groupId); } if (revalidation.empty) continue; try { host.signalGroup(processGroupId, "SIGKILL"); - signaled = true; + attempt.signaled = true; } catch (cause: unknown) { const code = cause instanceof Error && "code" in cause ? cause.code : undefined; if (code !== "ESRCH") - return mergeUnverifiedObservations( - { - cleaned: false, - reason: `owned process group signal failed: ${errorMessage(cause)}`, - }, - plan.unverified, - ); + return finish({ + cleaned: false, + reason: `owned process group signal failed: ${errorMessage(cause)}`, + }); } } if (plan.unresolved.length > 0) - return cleanupValidationFailure(plan.unresolved, plan.unverified); + return finish(cleanupValidationFailure(plan.unresolved)); if (ownership.sweepTokenOwnedProcesses === true) { let verification: ProcessCleanupResult; try { @@ -503,26 +526,18 @@ export const cleanupOwnedProcessGroup = async ( } } catch (cause: unknown) { rethrowProcessInspectionFailure(cause); - return mergeUnverifiedObservations( - { - cleaned: false, - reason: `final process ownership verification failed: ${errorMessage(cause)}`, - }, - plan.unverified, - ); + return finish({ + cleaned: false, + reason: `final process ownership verification failed: ${errorMessage(cause)}`, + }); } - if (!verification.cleaned) - return mergeUnverifiedObservations(verification, plan.unverified); - return mergeUnverifiedObservations({ cleaned: true, signaled }, [ + if (!verification.cleaned) return finish(verification); + return finish({ cleaned: true, signaled: attempt.signaled }, [ ...plan.unverified, ...(verification.unverified ?? []), ]); } - return { - cleaned: true, - signaled, - ...(plan.unverified.length === 0 ? {} : { unverified: plan.unverified }), - }; + return finish({ cleaned: true, signaled: attempt.signaled }); }; /** Verify that no live process still carries an owned capture run token. */ diff --git a/src/process/capture/ProcessCaptureCleanupOutcome.test.ts b/src/process/capture/ProcessCaptureCleanupOutcome.test.ts new file mode 100644 index 0000000000..1f55f7ab16 --- /dev/null +++ b/src/process/capture/ProcessCaptureCleanupOutcome.test.ts @@ -0,0 +1,313 @@ +import { expect, it } from "vitest"; +import { emptyProcessCapture } from "../../domain/process/processCapture.fixture.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { analysisErrorProjectionSchema } from "../../contracts/errorSchemas.js"; +import { ProcessCaptureError } from "./ProcessCaptureError.js"; +import { + PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN, + ProcessCaptureResourceScope, + resolveProcessResult, + createProcessCaptureObservationBuffer, + type ProcessCaptureCleanupOutcome, + type PendingProcessCapture, +} from "./ProcessCaptureLifecycle.js"; + +const pendingCapture = (): PendingProcessCapture => { + const { cleanup, ...capture } = emptyProcessCapture(); + void cleanup; + return capture; +}; + +// An injected ownership result exercises finalization without claiming host +// process/TTY behavior or sending a signal to a real process. +const observedCleanup = (signaled: boolean, rendererFailure?: () => boolean) => + new ProcessCaptureResourceScope().release({ + timers: new Set(), + terminal: { pid: 321 }, + renderer: + rendererFailure === undefined + ? undefined + : { + dispose: async () => { + if (rendererFailure()) throw new Error("renderer cleanup denied"); + }, + }, + runId: "cleanup-facts-fixture", + temporaryRoot: "/fixture/root", + host: { + platform: "linux", + cleanupProcessGroup: async () => ({ cleaned: true, signaled }), + removeTemporaryRoot: async () => undefined, + }, + }); + +const intervention = PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN; + +it.each(["capture", "observations"] as const)( + "retains a producer's known signal after failed cleanup in %s", + async (shape) => { + let recovered = false; + const cleanup = await new ProcessCaptureResourceScope().release({ + timers: new Set(), + terminal: { pid: 321 }, + renderer: undefined, + runId: "failed-cleanup-signal-fixture", + temporaryRoot: "/fixture/root", + host: { + platform: "linux", + cleanupProcessGroup: async () => + recovered + ? { cleaned: true, signaled: false } + : { + cleaned: false, + signaled: true, + reason: "post-signal verification failed", + }, + removeTemporaryRoot: async () => undefined, + }, + }); + const observations = createProcessCaptureObservationBuffer({ + frames: [], + interactions: [], + samples: [], + eventJournal: [], + before: undefined, + }); + const failure = new Error("original terminal failure"); + try { + resolveProcessResult( + shape === "capture" ? pendingCapture() : undefined, + failure, + cleanup, + shape === "observations" ? observations : undefined, + ); + throw new Error("Expected incomplete cleanup"); + } catch (cause: unknown) { + if (!(cause instanceof ProcessCaptureError)) throw cause; + expect(cause.cause).toBe(failure); + expect(cause.cleanupIncomplete).toBe(true); + expect(cause.cleanupReport?.owned_process_group).toEqual({ + state: "unverified", + reason: "post-signal verification failed", + }); + const partial = cause.partialObservation; + if (partial === undefined) + throw new Error("Expected partial observations"); + const facts = + "capture" in partial ? partial.capture : partial.observations; + expect(facts.residual_unknowns ?? []).toContainEqual(intervention); + expect( + analysisErrorProjectionSchema.parse(projectAnalysisError(cause)), + ).toMatchObject({ + details: { + partial_observation: { + [shape]: { + residual_unknowns: expect.arrayContaining([intervention]), + }, + }, + }, + }); + if ("capture" in partial) + expect(partial.capture.settlement.cleanup_outcome).toBe("failed"); + const historical = JSON.stringify(partial); + recovered = true; + await cleanup.cleanupLease?.release(); + expect(cause.cleanupIncomplete).toBe(false); + expect(JSON.stringify(cause.partialObservation)).toBe(historical); + } finally { + recovered = true; + await cleanup.cleanupLease?.release(); + } + }, +); + +it("reports cleanup intervention even when settlement had already quiesced", async () => { + const cleanup = await observedCleanup(true); + const capture = resolveProcessResult(pendingCapture(), undefined, cleanup); + expect(capture.settlement).toMatchObject({ + state: "quiesced", + cleanup_outcome: "cleaned", + }); + expect(capture.residual_unknowns).toContainEqual(intervention); + expect(capture.cleanup.owned_process_group).toBe("verified"); +}); + +it("keeps non-signaling cleanup distinct from intervention", async () => { + const cleanup = await observedCleanup(false); + const capture = resolveProcessResult(pendingCapture(), undefined, cleanup); + expect(capture.settlement.cleanup_outcome).toBe("not_required"); + expect(capture.residual_unknowns).not.toContainEqual(intervention); +}); + +it("updates a validated capture once and does not duplicate intervention facts", async () => { + const cleanup = await observedCleanup(true); + const capture = resolveProcessResult( + emptyProcessCapture(), + undefined, + cleanup, + ); + expect(capture.settlement.cleanup_outcome).toBe("cleaned"); + const repeated = resolveProcessResult(capture, undefined, cleanup); + expect(repeated.residual_unknowns).toEqual(capture.residual_unknowns); + expect( + repeated.residual_unknowns.filter( + (unknown) => + unknown.scope === intervention.scope && + unknown.reason === intervention.reason, + ), + ).toHaveLength(1); +}); + +it("retains intervention in partial facts without replacing the execution failure", async () => { + const cleanup = await observedCleanup(true); + const failure = new Error("final snapshot failed"); + try { + resolveProcessResult(pendingCapture(), failure, cleanup); + throw new Error("Expected the execution failure"); + } catch (cause: unknown) { + if (!(cause instanceof ProcessCaptureError)) throw cause; + expect(cause.cause).toBe(failure); + expect(cause.cleanupIncomplete).toBe(false); + expect(cause.partialObservation).toMatchObject({ + capture: { + settlement: { cleanup_outcome: "cleaned" }, + residual_unknowns: expect.arrayContaining([intervention]), + }, + }); + } +}); + +it("keeps failed cleanup authoritative while retaining its known intervention", async () => { + let rendererFails = true; + const cleanup = await observedCleanup(true, () => rendererFails); + try { + resolveProcessResult(pendingCapture(), undefined, cleanup); + throw new Error("Expected incomplete cleanup"); + } catch (cause: unknown) { + if (!(cause instanceof ProcessCaptureError)) throw cause; + expect(cause.cleanupIncomplete).toBe(true); + expect(cause.partialObservation).toMatchObject({ + capture: { + settlement: { cleanup_outcome: "failed" }, + residual_unknowns: expect.arrayContaining([intervention]), + }, + }); + } finally { + rendererFails = false; + await cleanup.cleanupLease?.release(); + } +}); + +it("does not invent intervention facts for a historical report", async () => { + const observed = await observedCleanup(true); + const historical: ProcessCaptureCleanupOutcome = { + report: observed.report, + cleanupLease: undefined, + }; + const capture = resolveProcessResult( + emptyProcessCapture(), + undefined, + historical, + ); + expect(capture.settlement.cleanup_outcome).toBe("not_required"); + expect(capture.residual_unknowns).not.toContainEqual(intervention); +}); + +it("preserves saved intervention when a later report has no signal fact", async () => { + const cleanup = await observedCleanup(true); + const capture = resolveProcessResult(pendingCapture(), undefined, cleanup); + const historical: ProcessCaptureCleanupOutcome = { + report: cleanup.report, + cleanupLease: undefined, + }; + const restored = resolveProcessResult(capture, undefined, historical); + expect(restored.settlement).toEqual(capture.settlement); + expect(restored.residual_unknowns).toEqual(capture.residual_unknowns); +}); + +it.each([false, true])( + "retains intervention before a capture exists, including failed cleanup: %s", + async (cleanupFails) => { + let rendererFails = cleanupFails; + const cleanup = await observedCleanup(true, () => rendererFails); + const observations = createProcessCaptureObservationBuffer({ + frames: [{ sequence: 0, at_ms: 0, data: "collected terminal output" }], + interactions: [], + samples: [], + eventJournal: [], + before: undefined, + }); + observations.settlement = { + state: "available", + value: { state: "quiesced", elapsed_ms: 12 }, + }; + const failure = new Error("final filesystem snapshot failed"); + try { + resolveProcessResult(undefined, failure, cleanup, observations); + throw new Error("Expected incomplete observations"); + } catch (cause: unknown) { + if (!(cause instanceof ProcessCaptureError)) throw cause; + expect(cause.cause).toBe(failure); + expect(cause.cleanupIncomplete).toBe(cleanupFails); + expect(cause.partialObservation).toMatchObject({ + observations: { + frames: observations.frames, + settlement: observations.settlement, + filesystem_snapshots: { + after: { state: "unavailable" }, + }, + residual_unknowns: expect.arrayContaining([intervention]), + }, + }); + expect( + analysisErrorProjectionSchema.parse(projectAnalysisError(cause)), + ).toMatchObject({ + details: { + partial_observation: { + observations: { + residual_unknowns: expect.arrayContaining([intervention]), + }, + }, + }, + }); + } finally { + rendererFails = false; + await cleanup.cleanupLease?.release(); + } + }, +); + +it.each([false, undefined])( + "does not infer intervention for incomplete observations from %s", + async (signaled) => { + const observed = await observedCleanup(false); + const cleanup: ProcessCaptureCleanupOutcome = { + report: observed.report, + cleanupLease: observed.cleanupLease, + ...(signaled === undefined ? {} : { ownedProcessesSignaled: signaled }), + }; + const observations = createProcessCaptureObservationBuffer({ + frames: [], + interactions: [], + samples: [], + eventJournal: [], + before: undefined, + }); + try { + resolveProcessResult( + undefined, + new Error("terminal read failed"), + cleanup, + observations, + ); + throw new Error("Expected incomplete observations"); + } catch (cause: unknown) { + if (!(cause instanceof ProcessCaptureError)) throw cause; + const partial = cause.partialObservation; + if (partial === undefined || !("observations" in partial)) + throw new Error("Expected incomplete observations"); + expect(partial.observations).not.toHaveProperty("residual_unknowns"); + expect(partial.observations.settlement.state).toBe("unavailable"); + } + }, +); diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index ea1029b8ae..d45047fba3 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -31,6 +31,7 @@ import { } from "./ProcessCaptureRead.js"; import { classifyFilesystemEffects } from "./ProcessFilesystemEffects.js"; import { + cleanupFailureFollowedSignal, cleanupOwnedProcessGroup, type OwnedProcessGroup, type ProcessOwnershipBaseline, @@ -1096,12 +1097,18 @@ const releaseCapturedProcessGroup = async ( { readonly sweepTokenOwnedProcesses: true } >, host: ProcessCaptureCleanupHost, -): Promise => { +): Promise<{ + readonly report: ProcessCaptureCleanupReport["owned_process_group"]; + readonly signaled: boolean | undefined; +}> => { if (host.platform === "win32") return { - state: "unverified", - reason: - "owned process cleanup is unverifiable on Windows without process-job authority", + report: { + state: "unverified", + reason: + "owned process cleanup is unverifiable on Windows without process-job authority", + }, + signaled: undefined, }; let outcome: ProcessCaptureCleanupReport["owned_process_group"] = { state: "cleaned", @@ -1115,19 +1122,23 @@ const releaseCapturedProcessGroup = async ( unverified.push(...(cleaned.unverified ?? [])); if (!cleaned.cleaned) outcome = { state: "unverified", reason: cleaned.reason }; - return unverified.length === 0 - ? outcome - : { - ...outcome, - unverified_processes: [ - ...new Map( - unverified.map(({ pid, diagnostic }) => [ - `${String(pid)}:${diagnostic}`, - { pid, reason: diagnostic }, - ]), - ).values(), - ], - }; + return { + signaled: cleaned.signaled, + report: + unverified.length === 0 + ? outcome + : { + ...outcome, + unverified_processes: [ + ...new Map( + unverified.map(({ pid, diagnostic }) => [ + `${String(pid)}:${diagnostic}`, + { pid, reason: diagnostic }, + ]), + ).values(), + ], + }, + }; }; export interface ProcessCaptureResourceCleanupOptions { @@ -1148,6 +1159,8 @@ export interface ProcessCaptureResourceCleanupOptions { /** Historical cleanup facts and the exact authority that can confirm recovery. */ export interface ProcessCaptureCleanupOutcome { readonly report: ProcessCaptureCleanupReport; + /** Missing historical or failed-attempt facts do not establish no signaling. */ + readonly ownedProcessesSignaled?: boolean; /** Undefined means that only historical facts, not an owner, are available. */ readonly cleanupLease: AnalysisCleanupLease | undefined; /** Explicit failures observed by this cleanup owner, absent for plain historical reports. */ @@ -1168,6 +1181,7 @@ class ProcessCaptureResourceOwner { | ProcessCaptureCleanupReport["owned_process_group"] | undefined; #temporaryRoot: ProcessCaptureCleanupReport["temporary_root"] | undefined; + #ownedProcessesSignaled: boolean | undefined = false; constructor(private readonly options: ProcessCaptureResourceCleanupOptions) { this.#host = options.host ?? processCaptureCleanupHost; @@ -1249,29 +1263,30 @@ class ProcessCaptureResourceOwner { async release(): Promise { try { await this.cleanupLease.release(); - return { - report: this.report, - cleanupLease: this.cleanupLease, - ...(this.#cleanupInspections.size === 0 - ? {} - : { inspectionFailures: [...this.#cleanupInspections] }), - }; + return this.#outcome(this.report); } catch (cause: unknown) { if ( !(cause instanceof ProcessCaptureError) || cause.cleanupReport === undefined ) throw cause; - return { - report: cause.cleanupReport, - cleanupLease: this.cleanupLease, - ...(this.#cleanupInspections.size === 0 - ? {} - : { inspectionFailures: [...this.#cleanupInspections] }), - }; + return this.#outcome(cause.cleanupReport); } } + #outcome(report: ProcessCaptureCleanupReport): ProcessCaptureCleanupOutcome { + return { + report, + cleanupLease: this.cleanupLease, + ...(this.#ownedProcessesSignaled === undefined + ? {} + : { ownedProcessesSignaled: this.#ownedProcessesSignaled }), + ...(this.#cleanupInspections.size === 0 + ? {} + : { inspectionFailures: [...this.#cleanupInspections] }), + }; + } + async #releaseRenderer(): Promise { if ( this.options.renderer === undefined || @@ -1306,7 +1321,7 @@ class ProcessCaptureResourceOwner { return; } try { - this.#ownedProcessGroup = await releaseCapturedProcessGroup( + const outcome = await releaseCapturedProcessGroup( { runId: this.options.runId, leaderPid: this.options.terminal.pid, @@ -1322,7 +1337,14 @@ class ProcessCaptureResourceOwner { }, this.#host, ); + this.#ownedProcessGroup = outcome.report; + this.#recordSignaled(outcome.signaled); } catch (cause: unknown) { + // A thrown attempt keeps a signal it is known to have sent; otherwise + // whether it signaled before failing is unknown. + this.#recordSignaled( + cleanupFailureFollowedSignal(cause) ? true : undefined, + ); const normalized = normalizeCaptureFailure(cause); if ( normalized instanceof ProcessCaptureInspectionFailure && @@ -1346,6 +1368,17 @@ class ProcessCaptureResourceOwner { ); } + // Retain observed intervention across retries; an unknown attempt followed + // by a non-signaling retry cannot prove that no earlier signal was sent. + #recordSignaled(signaled: boolean | undefined): void { + this.#ownedProcessesSignaled = + this.#ownedProcessesSignaled === true || signaled === true + ? true + : this.#ownedProcessesSignaled === undefined || signaled === undefined + ? undefined + : false; + } + async #releaseTemporaryRoot(): Promise { if (this.#temporaryRoot?.state === "cleaned") return; const owned = this.#ownedProcessGroup; @@ -1677,6 +1710,61 @@ export interface TerminalCaptureRetention { readonly normalized_terminal: NormalizedTerminalRetention; } +export const PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN = { + scope: "process" as const, + reason: + "Owned-process cleanup sent termination signals. Later terminal and filesystem observations may omit work ended by this intervention.", +}; + +const cleanupResidualUnknowns = ( + unknowns: UnverifiedProcessCapture["residual_unknowns"], + signaled: boolean | undefined, +): UnverifiedProcessCapture["residual_unknowns"] => + signaled === true && + !unknowns.some( + (unknown) => + unknown.scope === PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN.scope && + unknown.reason === PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN.reason, + ) + ? [...unknowns, PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN] + : unknowns; + +/** Interpret settlement and observed cleanup action once for full/partial output. */ +export const processCaptureCleanupFacts = ( + capture: Pick< + PendingProcessCapture | ProcessCapture, + "settlement" | "residual_unknowns" + >, + outcome: { + readonly ownedProcessesSignaled: boolean | undefined; + }, +): Pick => ({ + settlement: + capture.settlement.state === "quiesced" + ? { + ...capture.settlement, + cleanup_outcome: + outcome.ownedProcessesSignaled === true || + ("cleanup_outcome" in capture.settlement && + capture.settlement.cleanup_outcome === "cleaned") + ? "cleaned" + : "not_required", + } + : { + ...capture.settlement, + cleanup_outcome: + outcome.ownedProcessesSignaled !== true && + "cleanup_outcome" in capture.settlement && + capture.settlement.cleanup_outcome === "failed" + ? "failed" + : "cleaned", + }, + residual_unknowns: cleanupResidualUnknowns( + capture.residual_unknowns, + outcome.ownedProcessesSignaled, + ), +}); + export const resolveProcessResult = ( capture: PendingProcessCapture | ProcessCapture | undefined, executionFailure: unknown, @@ -1703,7 +1791,17 @@ export const resolveProcessResult = ( if (capture === undefined) { if (normalizedObservations !== undefined) { const partialResult = partialProcessCaptureObservationSchema.safeParse({ - observations: normalizedObservations, + observations: { + ...normalizedObservations, + ...(cleanupOutcome.ownedProcessesSignaled === true + ? { + residual_unknowns: cleanupResidualUnknowns( + normalizedObservations.residual_unknowns ?? [], + cleanupOutcome.ownedProcessesSignaled, + ), + } + : {}), + }, cleanup, execution_failure: executionFailureReason ?? null, }); @@ -1718,17 +1816,18 @@ export const resolveProcessResult = ( return data; })() : capture; - const settlement = - cleanupFailure !== undefined - ? { ...capture.settlement, cleanup_outcome: "failed" as const } - : capture.settlement.state === "quiesced" - ? { - ...capture.settlement, - cleanup_outcome: "not_required" as const, - } - : { ...capture.settlement, cleanup_outcome: "cleaned" as const }; + const cleanupFacts = processCaptureCleanupFacts(capture, { + ownedProcessesSignaled: cleanupOutcome.ownedProcessesSignaled, + }); const partialResult = partialProcessCaptureObservationSchema.safeParse({ - capture: { ...captureData, settlement }, + capture: { + ...captureData, + ...cleanupFacts, + settlement: + cleanupFailure === undefined + ? cleanupFacts.settlement + : { ...cleanupFacts.settlement, cleanup_outcome: "failed" }, + }, cleanup, execution_failure: executionFailureReason ?? null, }); @@ -1812,18 +1911,23 @@ export const resolveProcessResult = ( if (capture === undefined) throw new ProcessCaptureError("process capture produced no result"); const unverifiedProcesses = cleanup.owned_process_group.unverified_processes; - if ("cleanup" in capture && unverifiedProcesses === undefined) return capture; - - const settlement: UnverifiedProcessCapture["settlement"] = - capture.settlement.state === "quiesced" - ? { ...capture.settlement, cleanup_outcome: "not_required" } - : { ...capture.settlement, cleanup_outcome: "cleaned" }; + const cleanupFacts = processCaptureCleanupFacts(capture, { + ownedProcessesSignaled: cleanupOutcome.ownedProcessesSignaled, + }); + if ( + "cleanup" in capture && + unverifiedProcesses === undefined && + capture.settlement.cleanup_outcome === + cleanupFacts.settlement.cleanup_outcome && + capture.residual_unknowns === cleanupFacts.residual_unknowns + ) + return capture; try { return parseProcessCapture({ ...capture, - settlement, + ...cleanupFacts, residual_unknowns: [ - ...capture.residual_unknowns, + ...cleanupFacts.residual_unknowns, ...(unverifiedProcesses ?? []).map(({ pid, reason }) => ({ scope: "process" as const, reason: `Ownership of unrelated process ${String(pid)} could not be verified; it was left untouched: ${reason}`, diff --git a/src/process/capture/ProcessHarness.test.ts b/src/process/capture/ProcessHarness.test.ts index 3743042559..3b8194620b 100644 --- a/src/process/capture/ProcessHarness.test.ts +++ b/src/process/capture/ProcessHarness.test.ts @@ -9,6 +9,7 @@ import { } from "./ProcessHarness.js"; import { settleProcessCaptureJournal, + PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN, ProcessCaptureResourceScope, createProcessCaptureObservationBuffer, prepareProcessCapture, @@ -479,6 +480,7 @@ it("retains unrelated withheld processes through cleanup, capture, and JSON", as }); expect(capture.residual_unknowns).toEqual([ ...original.residual_unknowns, + PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN, { scope: "process", reason: expect.stringContaining("900 could not be verified"), @@ -712,45 +714,67 @@ it("retains completed observations when finalization fails after clean cleanup", ).toBe(true); }); -it("preserves cancellation while projecting observations and successful cleanup", () => { - const capture = parseProcessCapture({ - ...accountFullyObservedProcessCapture({ - ...emptyProcessCapture(), - frames: [{ sequence: 0, at_ms: 0, data: "observed before cancellation" }], - }), - event_journal: [], - }); - const cleanup = { - owned_process_group: { state: "cleaned" as const, reason: null }, - terminal_renderer: { state: "cleaned" as const, reason: null }, - temporary_root: { state: "cleaned" as const, reason: null }, - }; - let error: ProcessCaptureError | undefined; - try { - resolveProcessResult( - capture, - processCaptureCancelled(), - historicalCleanupOutcome(cleanup), - ); - } catch (cause: unknown) { - if (!(cause instanceof ProcessCaptureError)) throw cause; - error = cause; - } - if (error === undefined) throw new Error("expected cancellation"); - - expect(projectAnalysisError(error)).toMatchObject({ - code: "cancelled", - message: expect.stringMatching(/process capture.*cancelled.*again/iu), - details: { - operation: "process_capture", - cleanup: "complete", - cleanup_report: cleanup, - partial_observation: { - capture: { frames: [{ data: "observed before cancellation" }] }, +it.each([false, true])( + "preserves finalization cancellation alongside cleanup intervention=%s", + async (signaled) => { + const capture = parseProcessCapture({ + ...accountFullyObservedProcessCapture({ + ...emptyProcessCapture(), + frames: [ + { sequence: 0, at_ms: 0, data: "observed before cancellation" }, + ], + }), + event_journal: [], + }); + const cleanup = await new ProcessCaptureResourceScope().release({ + timers: new Set(), + terminal: { pid: 321 }, + renderer: undefined, + runId: "finalization-cancellation", + temporaryRoot: "/fixture/root", + host: { + platform: "linux", + cleanupProcessGroup: async () => ({ cleaned: true, signaled }), + removeTemporaryRoot: async () => undefined, }, - }, - }); -}); + }); + const reason = { request: "cancel after resource cleanup" }; + let error: ProcessCaptureError | undefined; + try { + resolveProcessResult(capture, processCaptureCancelled(reason), cleanup); + } catch (cause: unknown) { + if (!(cause instanceof ProcessCaptureError)) throw cause; + error = cause; + } + if (error === undefined) throw new Error("expected cancellation"); + + expect(error.cause).toBe(reason); + expect(projectAnalysisError(error)).toMatchObject({ + code: "cancelled", + message: expect.stringMatching(/process capture.*cancelled.*again/iu), + details: { + operation: "process_capture", + cleanup: "complete", + cleanup_report: cleanup.report, + partial_observation: { + capture: { + frames: [{ data: "observed before cancellation" }], + settlement: { + cleanup_outcome: signaled ? "cleaned" : "not_required", + }, + ...(signaled + ? { + residual_unknowns: expect.arrayContaining([ + PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN, + ]), + } + : {}), + }, + }, + }, + }); + }, +); it("fails closed on Windows before resolving or launching scenario paths", async () => { const scenario = processScenarioSchema.parse({ diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.ts index aa765e1949..048b6174e7 100644 --- a/src/process/capture/ProcessHarness.ts +++ b/src/process/capture/ProcessHarness.ts @@ -54,6 +54,7 @@ import { observeSettlement, prepareProcessCapture, resolveProcessResult, + processCaptureCleanupFacts, settleProcessCaptureJournal, type ProcessCaptureCleanupHost, type ProcessCaptureObservationBuffer, @@ -567,13 +568,11 @@ const finishProcessRun = async (options: { const cleanupFailure = cleanupReportFailure(cleanup); let verifiedCapture: ProcessCapture | undefined; if (capture !== undefined && cleanupFailure === undefined) { - const settlement: UnverifiedProcessCapture["settlement"] = - capture.settlement.state === "quiesced" - ? { ...capture.settlement, cleanup_outcome: "not_required" } - : { ...capture.settlement, cleanup_outcome: "cleaned" }; const completedCapture: UnverifiedProcessCapture = { ...capture, - settlement, + ...processCaptureCleanupFacts(capture, { + ownedProcessesSignaled: cleanupOutcome.ownedProcessesSignaled, + }), cleanup: { owned_process_group: "verified", temporary_root: "removed", diff --git a/tests/boundary/mcp/processCaptureCleanupSchema.test.ts b/tests/boundary/mcp/processCaptureCleanupSchema.test.ts new file mode 100644 index 0000000000..6e7038c81e --- /dev/null +++ b/tests/boundary/mcp/processCaptureCleanupSchema.test.ts @@ -0,0 +1,78 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { afterAll, expect, it, onTestFinished } from "vitest"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; +import { emptyProcessCapture } from "../../../src/domain/process/processCapture.fixture.js"; +import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; +import { createProcessCaptureEvidence } from "../../../src/application/process/ProcessEvidence.js"; +import { processScenarioSchema } from "../../../src/domain/process/processScenario.js"; + +const validation = createMcpSchemaValidation(); +afterAll(() => validation.clear()); + +it.each(["full", "compact"] as const)( + "advertises quiesced cleanup intervention without rejecting saved captures (%s)", + async (profile) => { + const session = createTestBinarySession(() => { + throw new Error("Schema discovery must not start a provider"); + }); + const server = createServer( + { kind: "session", session }, + { + environment: { + ...process.env, + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", + }, + }, + ); + const client = new Client( + { name: "capture-cleanup-schema", version: "1" }, + { jsonSchemaValidator: validation }, + ); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const schemas = await advertisedMcpToolSchemas( + client, + "capture_process_scenario", + validation.assertions, + ); + const saved = emptyProcessCapture(); + const scenario = processScenarioSchema.parse({ + executable: "fixture-program", + }); + const intervened = parseProcessCapture({ + ...saved, + settlement: { ...saved.settlement, cleanup_outcome: "cleaned" }, + residual_unknowns: [ + { scope: "process", reason: "Fixture cleanup intervention" }, + ], + }); + for (const capture of [saved, intervened]) { + const evidence = createProcessCaptureEvidence(scenario, capture); + expect( + schemas.ajv.validate(schemas.outputSchema, evidence), + JSON.stringify(schemas.ajv.errors), + ).toBe(true); + } + const evidence = createProcessCaptureEvidence(scenario, intervened); + expect( + schemas.ajv.validate(schemas.outputSchema, { + ...evidence, + normalized_result: { + ...intervened, + settlement: { ...intervened.settlement, cleanup_outcome: "invented" }, + }, + }), + ).toBe(false); + }, +); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index 5b1527d26d..c34b01a4c1 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -24,6 +24,7 @@ import { snapshotRoots } from "../../../src/process/capture/FilesystemSnapshot.j import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js"; import { expectUnverifiedHostCleanup } from "../../support/hostCleanup.js"; import { + PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN, ProcessCaptureResourceScope, type ProcessCaptureCleanupHost, observeLaunchedExecutable, @@ -313,10 +314,15 @@ itWithCaptureCapability( ); if (cleanupIncomplete) expect(capture.settlement.cleanup_outcome).toBe("failed"); - else + else { expect(result.ok && result.value.cleanup.owned_process_group).toBe( "verified", ); + expect(capture.settlement.cleanup_outcome).toBe("cleaned"); + expect(capture.residual_unknowns).toContainEqual( + PROCESS_CAPTURE_CLEANUP_INTERVENTION_UNKNOWN, + ); + } await waitForProcessExit(ownedPid); const unownedState = await processState(unownedPid); expect(unownedState).toBeDefined();