mirror of
https://github.com/morluto/rea.git
synced 2026-11-18 04:34:13 +00:00
fix(gdb): retain rejected shutdowns for retry
This commit is contained in:
1 parent
7557e39dcd
commit
f89b03e738
2 files changed
+214
-190
No files matched your search
+64
-100
@@ -1789,7 +1789,7 @@ export class GdbSessionManager {
|
||||
|
||||
pending.reject(
|
||||
new GdbProtocolFailureError(message, {
|
||||
command: pending.command,
|
||||
command: pending.evidenceCommand,
|
||||
mi: "",
|
||||
...snapshotGdbCommandRecords(pending.lines),
|
||||
output_truncated: pending.outputTruncated,
|
||||
@@ -1856,7 +1856,12 @@ export class GdbSessionManager {
|
||||
): ReturnType<typeof stopOwnedDebuggerProcess> {
|
||||
session.cleanupStarted = true;
|
||||
if (session.stopAttempt !== undefined) return session.stopAttempt;
|
||||
const attempt = this.#stopProcess(session, "GDB")
|
||||
const attempt = Promise.resolve()
|
||||
.then(() => this.#stopProcess(session, "GDB"))
|
||||
.catch((cause: unknown) => ({
|
||||
status: "incomplete" as const,
|
||||
reason: `Owned GDB shutdown failed: ${gdbErrorMessage(cause)}`,
|
||||
}))
|
||||
.then((stopped) => {
|
||||
if (stopped.status !== "incomplete") {
|
||||
for (const failure of session.cleanupFailures)
|
||||
@@ -1971,7 +1976,7 @@ export class GdbSessionManager {
|
||||
clearTimeout(pending.timer);
|
||||
pending.reject(
|
||||
new GdbCommandProcessExitError({
|
||||
command: pending.command,
|
||||
command: pending.evidenceCommand,
|
||||
mi: "",
|
||||
...snapshotGdbCommandRecords(pending.lines),
|
||||
process_exit_observed: true,
|
||||
@@ -2080,64 +2085,43 @@ export class GdbSessionManager {
|
||||
"The unrestricted GDB console can access local shell, filesystem, network, and target controls.",
|
||||
],
|
||||
});
|
||||
void this.#stopDebuggerOnly(session).then(
|
||||
(stopped) => {
|
||||
if (
|
||||
stopped.status === "incomplete" &&
|
||||
operation === "start_gdb_session"
|
||||
) {
|
||||
pending.reject(
|
||||
new AnalysisTimeoutError(operation, this.#commandTimeoutMs, {
|
||||
cause: timeout,
|
||||
cleanup: { reason: stopped.reason, resources: [session.id] },
|
||||
partialObservation: evidence,
|
||||
...copiedTimeoutGuidance(timeout),
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
void this.#stopDebuggerOnly(session).then((stopped) => {
|
||||
if (
|
||||
stopped.status === "incomplete" &&
|
||||
operation === "start_gdb_session"
|
||||
) {
|
||||
pending.reject(
|
||||
stopped.status === "incomplete"
|
||||
? new AnalysisCapabilityUnavailableError(
|
||||
"gdb",
|
||||
"gdb_console",
|
||||
`GDB command output could not be correlated and its owned process could not be confirmed stopped: ${stopped.reason}`,
|
||||
{
|
||||
cause: timeout,
|
||||
cleanup: {
|
||||
reason: stopped.reason,
|
||||
resources: [session.id],
|
||||
},
|
||||
partialObservation: evidence,
|
||||
},
|
||||
)
|
||||
: new GdbCommandTimeoutError(
|
||||
partial,
|
||||
evidence,
|
||||
this.#commandTimeoutMs,
|
||||
),
|
||||
new AnalysisTimeoutError(operation, this.#commandTimeoutMs, {
|
||||
cause: timeout,
|
||||
cleanup: { reason: stopped.reason, resources: [session.id] },
|
||||
partialObservation: evidence,
|
||||
...copiedTimeoutGuidance(timeout),
|
||||
}),
|
||||
);
|
||||
},
|
||||
(cause: unknown) =>
|
||||
pending.reject(
|
||||
new AnalysisCapabilityUnavailableError(
|
||||
"gdb",
|
||||
"gdb_console",
|
||||
"GDB command output could not be correlated and owned process shutdown failed.",
|
||||
{
|
||||
cause: new AggregateError(
|
||||
[timeout, cause],
|
||||
"GDB timeout and cleanup both failed.",
|
||||
),
|
||||
cleanup: {
|
||||
reason: gdbErrorMessage(cause),
|
||||
resources: [session.id],
|
||||
return;
|
||||
}
|
||||
pending.reject(
|
||||
stopped.status === "incomplete"
|
||||
? new AnalysisCapabilityUnavailableError(
|
||||
"gdb",
|
||||
"gdb_console",
|
||||
`GDB command output could not be correlated and its owned process could not be confirmed stopped: ${stopped.reason}`,
|
||||
{
|
||||
cause: timeout,
|
||||
cleanup: {
|
||||
reason: stopped.reason,
|
||||
resources: [session.id],
|
||||
},
|
||||
partialObservation: evidence,
|
||||
},
|
||||
partialObservation: evidence,
|
||||
},
|
||||
),
|
||||
),
|
||||
);
|
||||
)
|
||||
: new GdbCommandTimeoutError(
|
||||
partial,
|
||||
evidence,
|
||||
this.#commandTimeoutMs,
|
||||
),
|
||||
);
|
||||
});
|
||||
}, this.#commandTimeoutMs);
|
||||
pending = {
|
||||
token,
|
||||
@@ -2193,54 +2177,34 @@ export class GdbSessionManager {
|
||||
session.pending.delete(token);
|
||||
session.protocolError =
|
||||
"A GDB MI command was cancelled; owned-session shutdown was requested because output correlation can no longer be guaranteed.";
|
||||
void this.#stopDebuggerOnly(session).then(
|
||||
(stopped) => {
|
||||
if (stopped.status !== "incomplete") {
|
||||
pending.reject(
|
||||
new AnalysisCancelledError("gdb_console", {
|
||||
cause: signal?.reason,
|
||||
partialObservation: evidence,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const cleanupFailure = this.#retainCleanupFailure(
|
||||
session,
|
||||
"gdb_console",
|
||||
stopped.reason,
|
||||
);
|
||||
void this.#stopDebuggerOnly(session).then((stopped) => {
|
||||
if (stopped.status !== "incomplete") {
|
||||
pending.reject(
|
||||
new AnalysisCancelledError("gdb_console", {
|
||||
cause: signal?.reason,
|
||||
cleanup: {
|
||||
reason: stopped.reason,
|
||||
resources: [session.id],
|
||||
},
|
||||
partialObservation: evidence,
|
||||
}).bindCleanupDependencies([cleanupFailure], {
|
||||
cleanupResources: [session.id],
|
||||
}),
|
||||
);
|
||||
},
|
||||
(cause: unknown) => {
|
||||
const reason = gdbErrorMessage(cause);
|
||||
const cleanupFailure = this.#retainCleanupFailure(
|
||||
session,
|
||||
"gdb_console",
|
||||
reason,
|
||||
{ cause },
|
||||
);
|
||||
pending.reject(
|
||||
new AnalysisCancelledError("gdb_console", {
|
||||
cause: signal?.reason,
|
||||
cleanup: { reason, resources: [session.id] },
|
||||
partialObservation: evidence,
|
||||
}).bindCleanupDependencies([cleanupFailure], {
|
||||
cleanupResources: [session.id],
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
const cleanupFailure = this.#retainCleanupFailure(
|
||||
session,
|
||||
"gdb_console",
|
||||
stopped.reason,
|
||||
);
|
||||
pending.reject(
|
||||
new AnalysisCancelledError("gdb_console", {
|
||||
cause: signal?.reason,
|
||||
cleanup: {
|
||||
reason: stopped.reason,
|
||||
resources: [session.id],
|
||||
},
|
||||
partialObservation: evidence,
|
||||
}).bindCleanupDependencies([cleanupFailure], {
|
||||
cleanupResources: [session.id],
|
||||
}),
|
||||
);
|
||||
});
|
||||
};
|
||||
signal?.addEventListener("abort", abortListener, { once: true });
|
||||
if (signal?.aborted) {
|
||||
|
||||
@@ -438,6 +438,35 @@ it("keeps a consumed startup failure primary when caller abort arrives during cl
|
||||
});
|
||||
});
|
||||
|
||||
it("retains failed startup ownership when stop rejects and resolves it on retry", async () => {
|
||||
const fake = createFakeSpawner((child, token) => {
|
||||
child.stdout.write(`${token}^error,msg="configuration rejected"\n`);
|
||||
});
|
||||
let recovered = false;
|
||||
const manager = managerWithFakeSpawner(fake.spawnProcess, {
|
||||
stopProcess: async (session, name) => {
|
||||
if (!recovered) throw new Error("startup stop rejected");
|
||||
return stopOwnedDebuggerProcess(session, name);
|
||||
},
|
||||
});
|
||||
const started = await manager.start();
|
||||
expect(started).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
cleanupIncomplete: true,
|
||||
cause: { _tag: "AnalysisCapabilityUnavailableError" },
|
||||
message: expect.stringContaining("configuration"),
|
||||
},
|
||||
});
|
||||
if (started.ok) return;
|
||||
expect(started.error.cleanupResources).toHaveLength(1);
|
||||
expect((await manager.retryCleanupFailures()).ok).toBe(false);
|
||||
recovered = true;
|
||||
expect((await manager.retryCleanupFailures()).ok).toBe(true);
|
||||
expect(started.error.cleanupIncomplete).toBe(false);
|
||||
expect(fake.children[0]?.killSignals).toEqual(["SIGTERM"]);
|
||||
});
|
||||
|
||||
it("cancels startup when aborted during the final readiness drain", async () => {
|
||||
const controller = new AbortController();
|
||||
const reason = "caller cancelled during readiness drain";
|
||||
@@ -3023,7 +3052,10 @@ it("retains unknown inferior ownership after GDB exits during a command", async
|
||||
error: {
|
||||
cleanupIncomplete: true,
|
||||
partialObservation: {
|
||||
parameters: { command: "exit during command" },
|
||||
raw_result: { command: "exit during command" },
|
||||
normalized_result: {
|
||||
command: "exit during command",
|
||||
process_exit_observed: true,
|
||||
completion_status: "unknown",
|
||||
},
|
||||
@@ -3182,36 +3214,46 @@ it("retains caller cancellation and cleanup authority after active-command stop
|
||||
expect(stopCalls).toBe(2);
|
||||
});
|
||||
|
||||
it("returns a tagged cleanup failure thrown while closing", async () => {
|
||||
const fake = createFakeSpawner();
|
||||
const manager = managerWithFakeSpawner(fake.spawnProcess, {
|
||||
stopProcess: async () => {
|
||||
throw new AnalysisTimeoutError("close_gdb_session", 17);
|
||||
},
|
||||
});
|
||||
const started = await expectGdbSessionStarted(manager);
|
||||
|
||||
const closed = await manager.close(started.value.session_id);
|
||||
|
||||
expect(closed).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "AnalysisTimeoutError", operation: "close_gdb_session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rethrows an unexpected non-domain cleanup failure while closing", async () => {
|
||||
const fake = createFakeSpawner();
|
||||
const manager = managerWithFakeSpawner(fake.spawnProcess, {
|
||||
stopProcess: async () => {
|
||||
throw new Error("unexpected cleanup failure");
|
||||
},
|
||||
});
|
||||
const started = await expectGdbSessionStarted(manager);
|
||||
|
||||
await expect(manager.close(started.value.session_id)).rejects.toThrow(
|
||||
"unexpected cleanup failure",
|
||||
);
|
||||
});
|
||||
it.each(["tagged", "unexpected"] as const)(
|
||||
"retains %s stop failures while closing and resolves them after retry",
|
||||
async (failureKind) => {
|
||||
const fake = createFakeSpawner();
|
||||
let recovered = false;
|
||||
const manager = managerWithFakeSpawner(fake.spawnProcess, {
|
||||
stopProcess: () => {
|
||||
if (recovered) {
|
||||
fake.children[0]?.exit(0, null);
|
||||
return Promise.resolve({ status: "stopped" });
|
||||
}
|
||||
// Include synchronous throws at this injected provider boundary.
|
||||
throw failureKind === "tagged"
|
||||
? new AnalysisTimeoutError("foreign_cleanup", 17)
|
||||
: new Error("unexpected cleanup failure");
|
||||
},
|
||||
});
|
||||
const started = await expectGdbSessionStarted(manager);
|
||||
const closed = await manager.close(started.value.session_id);
|
||||
expect(closed).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
_tag: "AnalysisCapabilityUnavailableError",
|
||||
operation: "close_gdb_session",
|
||||
cleanupIncomplete: true,
|
||||
cleanupResources: [started.value.session_id],
|
||||
message: expect.stringContaining(
|
||||
failureKind === "tagged"
|
||||
? "foreign_cleanup"
|
||||
: "unexpected cleanup failure",
|
||||
),
|
||||
},
|
||||
});
|
||||
expect((await manager.retryCleanupFailures()).ok).toBe(false);
|
||||
recovered = true;
|
||||
expect((await manager.retryCleanupFailures()).ok).toBe(true);
|
||||
expect(manager.status(started.value.session_id)).toBeUndefined();
|
||||
if (!closed.ok) expect(closed.error.cleanupIncomplete).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("returns partial command evidence when the active MI request times out after stop", async () => {
|
||||
const fake = createFakeSpawner((_child, token, command) => {
|
||||
@@ -3573,68 +3615,86 @@ it("completes an MI record split after a chunk without a newline", async () => {
|
||||
expect(result.value.result.records).toContain('=thread-created,id="1"');
|
||||
});
|
||||
|
||||
it("preserves the protocol failure when an incomplete stop adds uncertainty", async () => {
|
||||
let stopCalls = 0;
|
||||
const fake = createFakeSpawner();
|
||||
const manager = managerWithFakeSpawner(fake.spawnProcess, {
|
||||
stopProcess: async (session, name) => {
|
||||
stopCalls += 1;
|
||||
if (stopCalls === 1) {
|
||||
return {
|
||||
status: "incomplete",
|
||||
reason: "simulated protocol stop uncertainty",
|
||||
};
|
||||
}
|
||||
return stopOwnedDebuggerProcess(session, name);
|
||||
},
|
||||
});
|
||||
const started = await expectGdbSessionStarted(manager);
|
||||
const stdin = fake.children[0]?.stdin;
|
||||
if (stdin === null || stdin === undefined) return;
|
||||
Object.defineProperty(stdin, "write", {
|
||||
value: (
|
||||
_chunk: string,
|
||||
_encoding: BufferEncoding,
|
||||
callback: (error?: Error | null) => void,
|
||||
): boolean => {
|
||||
fake.children[0]?.stdout.write('~"protocol output before failure"\n');
|
||||
callback(new Error("simulated protocol write failure"));
|
||||
return false;
|
||||
},
|
||||
});
|
||||
|
||||
const failed = await manager.execute(started.value.session_id, "will-fail");
|
||||
|
||||
expect(failed.ok).toBe(false);
|
||||
if (!failed.ok) {
|
||||
expect(failed.error.message).toContain(
|
||||
"GDB stdin write failed: simulated protocol write failure",
|
||||
);
|
||||
expect(failed.error.message).toContain(
|
||||
"simulated protocol stop uncertainty",
|
||||
);
|
||||
expect(failed.error.cleanup).toEqual({
|
||||
reason: "simulated protocol stop uncertainty",
|
||||
resources: [started.value.session_id],
|
||||
});
|
||||
expect(failed.error.partialObservation).toMatchObject({
|
||||
predicate_type: "rea.debugger.console-observation",
|
||||
raw_result: {
|
||||
records: ['~"protocol output before failure"'],
|
||||
completion_status: "unknown",
|
||||
it.each(["incomplete", "rejected"] as const)(
|
||||
"preserves the protocol failure and cleanup owner when stop is %s",
|
||||
async (stopOutcome) => {
|
||||
let stopCalls = 0;
|
||||
const fake = createFakeSpawner();
|
||||
const manager = managerWithFakeSpawner(fake.spawnProcess, {
|
||||
stopProcess: async (session, name) => {
|
||||
stopCalls += 1;
|
||||
if (stopCalls === 1) {
|
||||
if (stopOutcome === "rejected")
|
||||
throw new Error("simulated protocol stop uncertainty");
|
||||
return {
|
||||
status: "incomplete",
|
||||
reason: "simulated protocol stop uncertainty",
|
||||
};
|
||||
}
|
||||
if (stopCalls === 3) {
|
||||
// Simulate later owner verification after recovery completed.
|
||||
fake.children[0]?.exit(0, null);
|
||||
return { status: "stopped" };
|
||||
}
|
||||
return stopOwnedDebuggerProcess(session, name);
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(manager.status(started.value.session_id)?.state).toBe("closed");
|
||||
const subsequent = await manager.execute(
|
||||
started.value.session_id,
|
||||
"must-not-run",
|
||||
);
|
||||
expect(subsequent).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "AnalysisInputError" },
|
||||
});
|
||||
});
|
||||
const started = await expectGdbSessionStarted(manager);
|
||||
const stdin = fake.children[0]?.stdin;
|
||||
if (stdin === null || stdin === undefined) return;
|
||||
Object.defineProperty(stdin, "write", {
|
||||
value: (
|
||||
_chunk: string,
|
||||
_encoding: BufferEncoding,
|
||||
callback: (error?: Error | null) => void,
|
||||
): boolean => {
|
||||
fake.children[0]?.stdout.write('~"protocol output before failure"\n');
|
||||
callback(new Error("simulated protocol write failure"));
|
||||
return false;
|
||||
},
|
||||
});
|
||||
|
||||
const failed = await manager.execute(started.value.session_id, "will-fail");
|
||||
|
||||
expect(failed.ok).toBe(false);
|
||||
if (!failed.ok) {
|
||||
expect(failed.error.message).toContain(
|
||||
"GDB stdin write failed: simulated protocol write failure",
|
||||
);
|
||||
expect(failed.error.message).toContain(
|
||||
"simulated protocol stop uncertainty",
|
||||
);
|
||||
expect(failed.error.cleanup).toEqual({
|
||||
reason: expect.stringContaining("simulated protocol stop uncertainty"),
|
||||
resources: [started.value.session_id],
|
||||
});
|
||||
expect(failed.error.partialObservation).toMatchObject({
|
||||
predicate_type: "rea.debugger.console-observation",
|
||||
raw_result: {
|
||||
command: "will-fail",
|
||||
records: ['~"protocol output before failure"'],
|
||||
completion_status: "unknown",
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(manager.status(started.value.session_id)?.state).toBe("closed");
|
||||
const subsequent = await manager.execute(
|
||||
started.value.session_id,
|
||||
"must-not-run",
|
||||
);
|
||||
expect(subsequent).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "AnalysisInputError" },
|
||||
});
|
||||
expect((await manager.retryCleanupFailures()).ok).toBe(false);
|
||||
expect(stopCalls).toBe(2);
|
||||
if (!failed.ok) expect(failed.error.cleanupIncomplete).toBe(true);
|
||||
expect((await manager.retryCleanupFailures()).ok).toBe(true);
|
||||
expect(stopCalls).toBe(3);
|
||||
expect(manager.status(started.value.session_id)).toBeUndefined();
|
||||
if (!failed.ok) expect(failed.error.cleanupIncomplete).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("drains buffered startup stdout before capturing early-exit Evidence", async () => {
|
||||
const children: FakeGdbProcess[] = [];
|
||||
|
||||
Reference in new issue
Block a user