From bf638b47f20a15fe060254b318e7d80418e7c8df Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:48:48 +0800 Subject: [PATCH] fix(native): keep a completed LLDB trace when the deadline elapses during cleanup (#11116) observe_native_calls no longer reports provider_timeout when LLDB exited within its deadline and the deadline elapsed during process cleanup. The completed trace is returned; unconfirmed cleanup is reported as cleanup_incomplete with the trace as its partial observation. --- src/native/LldbCallTracer.ts | 23 ++--- .../native/lldbCallTracerFailures.test.ts | 90 ++++++++++++++++++- 2 files changed, 99 insertions(+), 14 deletions(-) diff --git a/src/native/LldbCallTracer.ts b/src/native/LldbCallTracer.ts index a9849316ab..3a832cab98 100644 --- a/src/native/LldbCallTracer.ts +++ b/src/native/LldbCallTracer.ts @@ -23,6 +23,7 @@ import { AnalysisTimeoutError, cancelledBy, carriedReason, + type AnalysisTimeoutRecovery, } from "../domain/analysisErrorCore.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; import type { @@ -86,9 +87,7 @@ const LLDB_REQUIREMENT = const DEADLINE_GRACE_MS = 60_000; const DIAGNOSTIC_BYTES = 16 * 1024; -const lldbGraceRecovery = ( - durationMs: number, -): { readonly userMessage: string; readonly remediationAction: string } => { +const lldbGraceRecovery = (durationMs: number): AnalysisTimeoutRecovery => { const total = durationMs + DEADLINE_GRACE_MS; return { userMessage: `observe_native_calls reached its ${String(total)} ms server/provider deadline: duration_ms ${String(durationMs)} plus a fixed ${String(DEADLINE_GRACE_MS)} ms LLDB startup, symbol loading, and teardown grace. This is not the MCP client deadline.`, @@ -951,7 +950,7 @@ export class LldbCallTracer implements NativeCallTracer { stderr: stderr.value, terminated, }; - if (exited.lateTimeout === true) + if (exited.overran === true) return err( new AnalysisTimeoutError( OPERATION, @@ -1387,8 +1386,8 @@ type LldbExit = readonly targetLaunched: boolean; readonly targetIdentity: string | undefined; readonly cleanupFailure: string | undefined; - /** Internal deadline elapsed during cleanup after a physical LLDB exit. */ - readonly lateTimeout?: true; + /** The work budget elapsed before this physical LLDB exit was selected. */ + readonly overran?: true; readonly cleanupOwner?: ProviderProcessSupervisor; }) | { @@ -1450,19 +1449,23 @@ const runLldb = async ( processFailureBeforeInterruption !== undefined ? { kind: "exited" as const, ...processFailureBeforeInterruption } : outcome; - const stopped = await supervisor?.stop(); - retained = stopped?.status === "incomplete"; - const lateTimeout = + // The work budget is read when the exit is selected, before cleanup: an + // exit selected after it elapsed overran the budget, while a budget that + // elapses during cleanup keeps the exit. Unconfirmed cleanup is reported + // by `cleanupFailure`, never as a timeout. + const overran = selectedOutcome.kind === "exited" && deadline.interruption === "timeout" ? true : undefined; + const stopped = await supervisor?.stop(); + retained = stopped?.status === "incomplete"; return { ...selectedOutcome, targetLaunched: supervisor !== undefined, targetIdentity, cleanupFailure: stopped?.status === "incomplete" ? stopped.reason : undefined, - ...(lateTimeout === undefined ? {} : { lateTimeout }), + ...(overran === undefined ? {} : { overran }), ...(retained && supervisor !== undefined ? { cleanupOwner: supervisor } : {}), diff --git a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts index d7f63b26e0..a8da443d9f 100644 --- a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts +++ b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts @@ -1412,8 +1412,8 @@ describe("LLDB process exit at output-close loop completion", () => { ); }); -describe("LLDB late timeout publication", () => { - it("converts a late deadline to timeout at successful-result publication", async () => { +describe("LLDB work budget at exit selection", () => { + it("converts an exit selected after the work budget to a timeout", async () => { let runtime: string | undefined; const tracer = new LldbCallTracer( {}, @@ -1437,7 +1437,7 @@ describe("LLDB late timeout publication", () => { output: "", targetIdentity: undefined, cleanupFailure: undefined, - lateTimeout: true, + overran: true, }; }, async () => ok(fakeTool), @@ -1445,7 +1445,7 @@ describe("LLDB late timeout publication", () => { const result = await tracer.trace(request); if (runtime === undefined) throw new Error("LLDB fixture did not run"); - if (result.ok) throw new Error("Expected the late deadline to be retained"); + if (result.ok) throw new Error("Expected the overrun to be a timeout"); expect(result.error).toBeInstanceOf(AnalysisTimeoutError); if (!(result.error instanceof AnalysisTimeoutError)) return; expect(result.error.timeoutMs).toBeGreaterThan(request.input.duration_ms); @@ -1464,6 +1464,88 @@ describe("LLDB late timeout publication", () => { }); }); +describe("LLDB work budget elapsing during process cleanup", () => { + it + .skipIf(process.platform === "win32") + .each(["confirmed", "unconfirmed"] as const)( + "keeps the completed trace when the budget elapses during %s process cleanup", + async (cleanup) => { + const fixtureRoot = await mkdtemp( + join(tmpdir(), "rea-lldb-cleanup-deadline-"), + ); + const fakeLldb = join(fixtureRoot, "fake-lldb"); + const observation = `${JSON.stringify({ kind: "event", event: EVENT })}\n`; + await writeFile( + fakeLldb, + [ + `#!${process.execPath}`, + `const fs = require("node:fs");`, + `const command = process.argv.at(-1);`, + `if (!command?.startsWith("rea_trace ")) process.exit(91);`, + `const config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));`, + // This process stands in for the target; it has exited by the + // time the tracer proves target termination. + `fs.writeFileSync(config.pid_path, String(process.pid));`, + `fs.writeFileSync(config.observation_path, ${JSON.stringify(observation)});`, + `fs.writeFileSync(config.stdout_capture_path, "");`, + `fs.writeFileSync(config.stderr_capture_path, "");`, + `fs.writeFileSync(config.result_path, ${JSON.stringify(JSON.stringify(successfulTracerOutput))});`, + `process.exit(0);`, + "", + ].join("\n"), + { mode: 0o700 }, + ); + // The work budget is live until process cleanup starts, then elapsed. + let stopping = false; + const deadlineSpy = vi + .spyOn(ProviderStartupDeadline.prototype, "interruption", "get") + .mockImplementation(() => (stopping ? "timeout" : undefined)); + const originalStop = ProviderProcessSupervisor.prototype.stop; + const stopSpy = vi + .spyOn(ProviderProcessSupervisor.prototype, "stop") + .mockImplementation(async function ( + this: ProviderProcessSupervisor, + options?: ProviderProcessStopOptions, + ): Promise { + stopping = true; + const stopped = await originalStop.call(this, options); + return cleanup === "confirmed" + ? stopped + : { status: "incomplete", reason: "fixture cleanup unconfirmed" }; + }); + const tracer = new LldbCallTracer({}, undefined, async () => + ok(nativeResolvedToolFixture(fakeLldb, "c".repeat(64))), + ); + try { + const result = await tracer.trace(request); + + expect(stopping).toBe(true); + if (cleanup === "confirmed") { + if (!result.ok) throw result.error; + expect(result.value.run).toMatchObject({ status: "traced" }); + return; + } + if (result.ok) throw new Error("Expected unconfirmed cleanup"); + expect(result.error).not.toBeInstanceOf(AnalysisTimeoutError); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain("lldb-process-group"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cleanup_incomplete", + }); + expect(result.error.partialObservation).toMatchObject({ + coverage: { status: "partial", reason: "cleanup-failure" }, + events: [EVENT], + }); + } finally { + stopSpy.mockRestore(); + deadlineSpy.mockRestore(); + await tracer.close(); + await rm(fixtureRoot, { recursive: true, force: true }); + } + }, + ); +}); + describe("LLDB pre-target phase proof", () => { it("releases an invalid-process launch failure and permits the next complete observation", async () => { let launches = 0;