mirror of
https://github.com/morluto/rea.git
synced 2026-11-15 19:23:24 +00:00
fix(native): keep a completed LLDB trace when the deadline elapses during cleanup (#11116)
observe_native_calls no longer reports provider_timeout when LLDB exited within its deadline and the deadline elapsed during process cleanup. The completed trace is returned; unconfirmed cleanup is reported as cleanup_incomplete with the trace as its partial observation.
This commit is contained in:
1 parent
a8e472967b
commit
bf638b47f2
2 files changed
+99
-14
No files matched your search
@@ -23,6 +23,7 @@ import {
|
||||
AnalysisTimeoutError,
|
||||
cancelledBy,
|
||||
carriedReason,
|
||||
type AnalysisTimeoutRecovery,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";
|
||||
import type {
|
||||
@@ -86,9 +87,7 @@ const LLDB_REQUIREMENT =
|
||||
const DEADLINE_GRACE_MS = 60_000;
|
||||
const DIAGNOSTIC_BYTES = 16 * 1024;
|
||||
|
||||
const lldbGraceRecovery = (
|
||||
durationMs: number,
|
||||
): { readonly userMessage: string; readonly remediationAction: string } => {
|
||||
const lldbGraceRecovery = (durationMs: number): AnalysisTimeoutRecovery => {
|
||||
const total = durationMs + DEADLINE_GRACE_MS;
|
||||
return {
|
||||
userMessage: `observe_native_calls reached its ${String(total)} ms server/provider deadline: duration_ms ${String(durationMs)} plus a fixed ${String(DEADLINE_GRACE_MS)} ms LLDB startup, symbol loading, and teardown grace. This is not the MCP client deadline.`,
|
||||
@@ -951,7 +950,7 @@ export class LldbCallTracer implements NativeCallTracer {
|
||||
stderr: stderr.value,
|
||||
terminated,
|
||||
};
|
||||
if (exited.lateTimeout === true)
|
||||
if (exited.overran === true)
|
||||
return err(
|
||||
new AnalysisTimeoutError(
|
||||
OPERATION,
|
||||
@@ -1387,8 +1386,8 @@ type LldbExit =
|
||||
readonly targetLaunched: boolean;
|
||||
readonly targetIdentity: string | undefined;
|
||||
readonly cleanupFailure: string | undefined;
|
||||
/** Internal deadline elapsed during cleanup after a physical LLDB exit. */
|
||||
readonly lateTimeout?: true;
|
||||
/** The work budget elapsed before this physical LLDB exit was selected. */
|
||||
readonly overran?: true;
|
||||
readonly cleanupOwner?: ProviderProcessSupervisor;
|
||||
})
|
||||
| {
|
||||
@@ -1450,19 +1449,23 @@ const runLldb = async (
|
||||
processFailureBeforeInterruption !== undefined
|
||||
? { kind: "exited" as const, ...processFailureBeforeInterruption }
|
||||
: outcome;
|
||||
const stopped = await supervisor?.stop();
|
||||
retained = stopped?.status === "incomplete";
|
||||
const lateTimeout =
|
||||
// The work budget is read when the exit is selected, before cleanup: an
|
||||
// exit selected after it elapsed overran the budget, while a budget that
|
||||
// elapses during cleanup keeps the exit. Unconfirmed cleanup is reported
|
||||
// by `cleanupFailure`, never as a timeout.
|
||||
const overran =
|
||||
selectedOutcome.kind === "exited" && deadline.interruption === "timeout"
|
||||
? true
|
||||
: undefined;
|
||||
const stopped = await supervisor?.stop();
|
||||
retained = stopped?.status === "incomplete";
|
||||
return {
|
||||
...selectedOutcome,
|
||||
targetLaunched: supervisor !== undefined,
|
||||
targetIdentity,
|
||||
cleanupFailure:
|
||||
stopped?.status === "incomplete" ? stopped.reason : undefined,
|
||||
...(lateTimeout === undefined ? {} : { lateTimeout }),
|
||||
...(overran === undefined ? {} : { overran }),
|
||||
...(retained && supervisor !== undefined
|
||||
? { cleanupOwner: supervisor }
|
||||
: {}),
|
||||
|
||||
@@ -1412,8 +1412,8 @@ describe("LLDB process exit at output-close loop completion", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("LLDB late timeout publication", () => {
|
||||
it("converts a late deadline to timeout at successful-result publication", async () => {
|
||||
describe("LLDB work budget at exit selection", () => {
|
||||
it("converts an exit selected after the work budget to a timeout", async () => {
|
||||
let runtime: string | undefined;
|
||||
const tracer = new LldbCallTracer(
|
||||
{},
|
||||
@@ -1437,7 +1437,7 @@ describe("LLDB late timeout publication", () => {
|
||||
output: "",
|
||||
targetIdentity: undefined,
|
||||
cleanupFailure: undefined,
|
||||
lateTimeout: true,
|
||||
overran: true,
|
||||
};
|
||||
},
|
||||
async () => ok(fakeTool),
|
||||
@@ -1445,7 +1445,7 @@ describe("LLDB late timeout publication", () => {
|
||||
|
||||
const result = await tracer.trace(request);
|
||||
if (runtime === undefined) throw new Error("LLDB fixture did not run");
|
||||
if (result.ok) throw new Error("Expected the late deadline to be retained");
|
||||
if (result.ok) throw new Error("Expected the overrun to be a timeout");
|
||||
expect(result.error).toBeInstanceOf(AnalysisTimeoutError);
|
||||
if (!(result.error instanceof AnalysisTimeoutError)) return;
|
||||
expect(result.error.timeoutMs).toBeGreaterThan(request.input.duration_ms);
|
||||
@@ -1464,6 +1464,88 @@ describe("LLDB late timeout publication", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("LLDB work budget elapsing during process cleanup", () => {
|
||||
it
|
||||
.skipIf(process.platform === "win32")
|
||||
.each(["confirmed", "unconfirmed"] as const)(
|
||||
"keeps the completed trace when the budget elapses during %s process cleanup",
|
||||
async (cleanup) => {
|
||||
const fixtureRoot = await mkdtemp(
|
||||
join(tmpdir(), "rea-lldb-cleanup-deadline-"),
|
||||
);
|
||||
const fakeLldb = join(fixtureRoot, "fake-lldb");
|
||||
const observation = `${JSON.stringify({ kind: "event", event: EVENT })}\n`;
|
||||
await writeFile(
|
||||
fakeLldb,
|
||||
[
|
||||
`#!${process.execPath}`,
|
||||
`const fs = require("node:fs");`,
|
||||
`const command = process.argv.at(-1);`,
|
||||
`if (!command?.startsWith("rea_trace ")) process.exit(91);`,
|
||||
`const config = JSON.parse(fs.readFileSync(command.slice("rea_trace ".length), "utf8"));`,
|
||||
// This process stands in for the target; it has exited by the
|
||||
// time the tracer proves target termination.
|
||||
`fs.writeFileSync(config.pid_path, String(process.pid));`,
|
||||
`fs.writeFileSync(config.observation_path, ${JSON.stringify(observation)});`,
|
||||
`fs.writeFileSync(config.stdout_capture_path, "");`,
|
||||
`fs.writeFileSync(config.stderr_capture_path, "");`,
|
||||
`fs.writeFileSync(config.result_path, ${JSON.stringify(JSON.stringify(successfulTracerOutput))});`,
|
||||
`process.exit(0);`,
|
||||
"",
|
||||
].join("\n"),
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
// The work budget is live until process cleanup starts, then elapsed.
|
||||
let stopping = false;
|
||||
const deadlineSpy = vi
|
||||
.spyOn(ProviderStartupDeadline.prototype, "interruption", "get")
|
||||
.mockImplementation(() => (stopping ? "timeout" : undefined));
|
||||
const originalStop = ProviderProcessSupervisor.prototype.stop;
|
||||
const stopSpy = vi
|
||||
.spyOn(ProviderProcessSupervisor.prototype, "stop")
|
||||
.mockImplementation(async function (
|
||||
this: ProviderProcessSupervisor,
|
||||
options?: ProviderProcessStopOptions,
|
||||
): Promise<ProviderProcessStopResult> {
|
||||
stopping = true;
|
||||
const stopped = await originalStop.call(this, options);
|
||||
return cleanup === "confirmed"
|
||||
? stopped
|
||||
: { status: "incomplete", reason: "fixture cleanup unconfirmed" };
|
||||
});
|
||||
const tracer = new LldbCallTracer({}, undefined, async () =>
|
||||
ok(nativeResolvedToolFixture(fakeLldb, "c".repeat(64))),
|
||||
);
|
||||
try {
|
||||
const result = await tracer.trace(request);
|
||||
|
||||
expect(stopping).toBe(true);
|
||||
if (cleanup === "confirmed") {
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.run).toMatchObject({ status: "traced" });
|
||||
return;
|
||||
}
|
||||
if (result.ok) throw new Error("Expected unconfirmed cleanup");
|
||||
expect(result.error).not.toBeInstanceOf(AnalysisTimeoutError);
|
||||
expect(result.error.cleanupIncomplete).toBe(true);
|
||||
expect(result.error.cleanupResources).toContain("lldb-process-group");
|
||||
expect(projectAnalysisError(result.error)).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
});
|
||||
expect(result.error.partialObservation).toMatchObject({
|
||||
coverage: { status: "partial", reason: "cleanup-failure" },
|
||||
events: [EVENT],
|
||||
});
|
||||
} finally {
|
||||
stopSpy.mockRestore();
|
||||
deadlineSpy.mockRestore();
|
||||
await tracer.close();
|
||||
await rm(fixtureRoot, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("LLDB pre-target phase proof", () => {
|
||||
it("releases an invalid-process launch failure and permits the next complete observation", async () => {
|
||||
let launches = 0;
|
||||
|
||||
Reference in new issue
Block a user