fix(native): keep a completed LLDB trace when the deadline elapses during cleanup (#11116)

observe_native_calls no longer reports provider_timeout when LLDB exited
within its deadline and the deadline elapsed during process cleanup. The
completed trace is returned; unconfirmed cleanup is reported as
cleanup_incomplete with the trace as its partial observation.
This commit is contained in:
morluto authored and GitHub committed 2026-11-15 20:48:48 +08:00
1 parent a8e472967b
commit bf638b47f2
2 files changed
+99 -14

No files matched your search

+13 -10
View File
@@ -23,6 +23,7 @@ import {
AnalysisTimeoutError,
cancelledBy,
carriedReason,
type AnalysisTimeoutRecovery,
} from "../domain/analysisErrorCore.js";
import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";
import type {
@@ -86,9 +87,7 @@ const LLDB_REQUIREMENT =
const DEADLINE_GRACE_MS = 60_000;
const DIAGNOSTIC_BYTES = 16 * 1024;
const lldbGraceRecovery = (
durationMs: number,
): { readonly userMessage: string; readonly remediationAction: string } => {
const lldbGraceRecovery = (durationMs: number): AnalysisTimeoutRecovery => {
const total = durationMs + DEADLINE_GRACE_MS;
return {
userMessage: `observe_native_calls reached its ${String(total)} ms server/provider deadline: duration_ms ${String(durationMs)} plus a fixed ${String(DEADLINE_GRACE_MS)} ms LLDB startup, symbol loading, and teardown grace. This is not the MCP client deadline.`,
@@ -951,7 +950,7 @@ export class LldbCallTracer implements NativeCallTracer {
stderr: stderr.value,
terminated,
};
if (exited.lateTimeout === true)
if (exited.overran === true)
return err(
new AnalysisTimeoutError(
OPERATION,
@@ -1387,8 +1386,8 @@ type LldbExit =
readonly targetLaunched: boolean;
readonly targetIdentity: string | undefined;
readonly cleanupFailure: string | undefined;
/** Internal deadline elapsed during cleanup after a physical LLDB exit. */
readonly lateTimeout?: true;
/** The work budget elapsed before this physical LLDB exit was selected. */
readonly overran?: true;
readonly cleanupOwner?: ProviderProcessSupervisor;
})
| {
@@ -1450,19 +1449,23 @@ const runLldb = async (
processFailureBeforeInterruption !== undefined
? { kind: "exited" as const, ...processFailureBeforeInterruption }
: outcome;
const stopped = await supervisor?.stop();
retained = stopped?.status === "incomplete";
const lateTimeout =
// The work budget is read when the exit is selected, before cleanup: an
// exit selected after it elapsed overran the budget, while a budget that
// elapses during cleanup keeps the exit. Unconfirmed cleanup is reported
// by `cleanupFailure`, never as a timeout.
const overran =
selectedOutcome.kind === "exited" && deadline.interruption === "timeout"
? true
: undefined;
const stopped = await supervisor?.stop();
retained = stopped?.status === "incomplete";
return {
...selectedOutcome,
targetLaunched: supervisor !== undefined,
targetIdentity,
cleanupFailure:
stopped?.status === "incomplete" ? stopped.reason : undefined,
...(lateTimeout === undefined ? {} : { lateTimeout }),
...(overran === undefined ? {} : { overran }),
...(retained && supervisor !== undefined
? { cleanupOwner: supervisor }
: {}),
@@ -1412,8 +1412,8 @@ describe("LLDB process exit at output-close loop completion", () => {
);
});
describe("LLDB late timeout publication", () => {
it("converts a late deadline to timeout at successful-result publication", async () => {
describe("LLDB work budget at exit selection", () => {
it("converts an exit selected after the work budget to a timeout", async () => {
let runtime: string | undefined;
const tracer = new LldbCallTracer(
{},
@@ -1437,7 +1437,7 @@ describe("LLDB late timeout publication", () => {
output: "",
targetIdentity: undefined,
cleanupFailure: undefined,
lateTimeout: true,
overran: true,
};
},
async () => ok(fakeTool),
@@ -1445,7 +1445,7 @@ describe("LLDB late timeout publication", () => {
const result = await tracer.trace(request);
if (runtime === undefined) throw new Error("LLDB fixture did not run");
if (result.ok) throw new Error("Expected the late deadline to be retained");
if (result.ok) throw new Error("Expected the overrun to be a timeout");
expect(result.error).toBeInstanceOf(AnalysisTimeoutError);
if (!(result.error instanceof AnalysisTimeoutError)) return;
expect(result.error.timeoutMs).toBeGreaterThan(request.input.duration_ms);
@@ -1464,6 +1464,88 @@ describe("LLDB late timeout publication", () => {
});
});
describe("LLDB work budget elapsing during process cleanup", () => {
it
.skipIf(process.platform === "win32")
.each(["confirmed", "unconfirmed"] as const)(
"keeps the completed trace when the budget elapses during %s process cleanup",
async (cleanup) => {
const fixtureRoot = await mkdtemp(
join(tmpdir(), "rea-lldb-cleanup-deadline-"),
);
const fakeLldb = join(fixtureRoot, "fake-lldb");
const observation = `${JSON.stringify({ kind: "event", event: EVENT })}\n`;
await writeFile(
fakeLldb,
[
`#!${process.execPath}`,
`const fs = require("node:fs");`,
`const command = process.argv.at(-1);`,
`if (!command?.startsWith("rea_trace ")) process.exit(91);`,
`const config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));`,
// This process stands in for the target; it has exited by the
// time the tracer proves target termination.
`fs.writeFileSync(config.pid_path, String(process.pid));`,
`fs.writeFileSync(config.observation_path, ${JSON.stringify(observation)});`,
`fs.writeFileSync(config.stdout_capture_path, "");`,
`fs.writeFileSync(config.stderr_capture_path, "");`,
`fs.writeFileSync(config.result_path, ${JSON.stringify(JSON.stringify(successfulTracerOutput))});`,
`process.exit(0);`,
"",
].join("\n"),
{ mode: 0o700 },
);
// The work budget is live until process cleanup starts, then elapsed.
let stopping = false;
const deadlineSpy = vi
.spyOn(ProviderStartupDeadline.prototype, "interruption", "get")
.mockImplementation(() => (stopping ? "timeout" : undefined));
const originalStop = ProviderProcessSupervisor.prototype.stop;
const stopSpy = vi
.spyOn(ProviderProcessSupervisor.prototype, "stop")
.mockImplementation(async function (
this: ProviderProcessSupervisor,
options?: ProviderProcessStopOptions,
): Promise<ProviderProcessStopResult> {
stopping = true;
const stopped = await originalStop.call(this, options);
return cleanup === "confirmed"
? stopped
: { status: "incomplete", reason: "fixture cleanup unconfirmed" };
});
const tracer = new LldbCallTracer({}, undefined, async () =>
ok(nativeResolvedToolFixture(fakeLldb, "c".repeat(64))),
);
try {
const result = await tracer.trace(request);
expect(stopping).toBe(true);
if (cleanup === "confirmed") {
if (!result.ok) throw result.error;
expect(result.value.run).toMatchObject({ status: "traced" });
return;
}
if (result.ok) throw new Error("Expected unconfirmed cleanup");
expect(result.error).not.toBeInstanceOf(AnalysisTimeoutError);
expect(result.error.cleanupIncomplete).toBe(true);
expect(result.error.cleanupResources).toContain("lldb-process-group");
expect(projectAnalysisError(result.error)).toMatchObject({
code: "cleanup_incomplete",
});
expect(result.error.partialObservation).toMatchObject({
coverage: { status: "partial", reason: "cleanup-failure" },
events: [EVENT],
});
} finally {
stopSpy.mockRestore();
deadlineSpy.mockRestore();
await tracer.close();
await rm(fixtureRoot, { recursive: true, force: true });
}
},
);
});
describe("LLDB pre-target phase proof", () => {
it("releases an invalid-process launch failure and permits the next complete observation", async () => {
let launches = 0;