mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 14:05:49 +00:00
test(process): consolidate process capture finalization checks (#12785)
* test(process): consolidate capture finalization cases * test(process): check grace from the escalation barrier * test(process): verify cancellation preempts capture grace * test(process): join pending delivery when an assertion fails
This commit is contained in:
1 parent
c7353f4257
commit
c5907d3944
1 file changed
+101
-159
@@ -1,6 +1,8 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import type { IPty } from "@lydell/node-pty";
|
||||
|
||||
import { phaseArrival } from "../../../tests/support/lifecycleHarness.js";
|
||||
import { processScenarioSchema } from "../../domain/process/processScenario.js";
|
||||
import type { ProcessCaptureFinalization } from "../../domain/process/processCapture.js";
|
||||
import { awaitTerminalExit } from "./ProcessCaptureLifecycle.js";
|
||||
@@ -51,15 +53,17 @@ const fakeTerminal = (
|
||||
};
|
||||
};
|
||||
|
||||
const waitFor = async (condition: () => boolean): Promise<void> => {
|
||||
const deadline = Date.now() + 3_000;
|
||||
while (!condition()) {
|
||||
if (Date.now() > deadline) throw new Error("condition not reached");
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
const snapshots: ProcessCaptureFinalization[] = [];
|
||||
|
||||
const fakePollingClock = () => {
|
||||
vi.useFakeTimers({
|
||||
toFake: ["Date", "performance", "setInterval", "clearInterval"],
|
||||
});
|
||||
};
|
||||
|
||||
const snapshots: ProcessCaptureFinalization[] = [];
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
const observed = <Result extends { readonly unobserved?: boolean }>(
|
||||
result: Result,
|
||||
@@ -93,41 +97,31 @@ const start = (
|
||||
recordEvent: () => undefined,
|
||||
});
|
||||
|
||||
it("uses the monotonic running deadline after the wall clock moves backward", async () => {
|
||||
it("uses the monotonic running deadline and releases zero-interval polling after exit", async () => {
|
||||
fakePollingClock();
|
||||
const fake = fakeTerminal();
|
||||
const controller = new AbortController();
|
||||
const wallStart = Date.now();
|
||||
let reads = 0;
|
||||
const wall = vi.spyOn(Date, "now").mockImplementation(() => {
|
||||
reads += 1;
|
||||
return reads === 1 ? wallStart : wallStart - 60_000 + reads;
|
||||
});
|
||||
const kill = vi.spyOn(fake.terminal, "kill").mockImplementation(() => {
|
||||
queueMicrotask(() => fake.deliverExit({ exitCode: 0, signal: 9 }));
|
||||
});
|
||||
const fallback = setTimeout(() => controller.abort(), 1_500);
|
||||
try {
|
||||
const monotonicStarted = performance.now();
|
||||
const exit = observed(
|
||||
await start(fake, { timeout_ms: 100 }, controller.signal),
|
||||
);
|
||||
// The production clock rounds down to milliseconds, losing at most 1 ms.
|
||||
expect(performance.now() - monotonicStarted).toBeGreaterThanOrEqual(99);
|
||||
expect(exit.reason).toBe("timeout");
|
||||
expect(controller.signal.aborted).toBe(false);
|
||||
expect(kill).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
clearTimeout(fallback);
|
||||
kill.mockRestore();
|
||||
wall.mockRestore();
|
||||
}
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 0 });
|
||||
vi.setSystemTime(Date.now() - 60_000);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(80);
|
||||
expect(fake.signals).toEqual([]);
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
expect(fake.signals).toEqual(["terminal:SIGKILL"]);
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
expect(exit.reason).toBe("timeout");
|
||||
expect(exit.finalization).toBeUndefined();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("records attempts when a self-exit notification arrives after escalation", async () => {
|
||||
fakePollingClock();
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
await new Promise((resolve) => setTimeout(resolve, 150));
|
||||
// Let several more polls run after escalation before the PTY emits exit.
|
||||
await vi.advanceTimersByTimeAsync(350);
|
||||
expect(fake.signals).toEqual(["SIGTERM", "SIGKILL"]);
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -139,17 +133,10 @@ it("records attempts when a self-exit notification arrives after escalation", as
|
||||
expect(exit.signal, "the observed exit remains independent of attempts").toBe(
|
||||
0,
|
||||
);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal === "SIGKILL"),
|
||||
"the escalation is sent once, not on every poll",
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal === "SIGTERM"),
|
||||
"the finalization signal is sent once, not on every poll",
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("kills at once when cancelled during the finalization interval", async () => {
|
||||
fakePollingClock();
|
||||
const fake = fakeTerminal();
|
||||
const controller = new AbortController();
|
||||
const pending = start(
|
||||
@@ -157,9 +144,12 @@ it("kills at once when cancelled during the finalization interval", async () =>
|
||||
{ timeout_ms: 100, finalization_ms: 60_000 },
|
||||
controller.signal,
|
||||
);
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(fake.signals).toEqual(["SIGTERM"]);
|
||||
controller.abort();
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
// A few polls are enough for cancellation. The 60s interval stays open.
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(fake.signals).toEqual(["SIGTERM", "SIGKILL"]);
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -167,24 +157,16 @@ it("kills at once when cancelled during the finalization interval", async () =>
|
||||
expect(exit.reason, "cancellation replaces the initiating deadline").toBe(
|
||||
"cancelled",
|
||||
);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal === "SIGKILL"),
|
||||
"the abort branch itself sent one SIGKILL, long before the interval ended",
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
exit.finalization?.signals.map(({ signal }) => signal),
|
||||
"cancellation records both attempts",
|
||||
).toEqual(["SIGTERM", "SIGKILL"]);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal.startsWith("terminal:")),
|
||||
"cancellation during finalization never uses a bare terminal kill",
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("records escalation attempts alongside an observed SIGKILL exit", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGKILL");
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -200,15 +182,16 @@ it("records escalation attempts alongside an observed SIGKILL exit", async () =>
|
||||
9,
|
||||
);
|
||||
expect(
|
||||
exit.finalization?.elapsed_ms,
|
||||
exit.finalization?.signals[1]?.sent_at_ms,
|
||||
"the escalation waited for the whole interval",
|
||||
).toBeGreaterThanOrEqual(100);
|
||||
expect(exit.finalization?.exit_code).toBeNull();
|
||||
});
|
||||
|
||||
it("records a scripted SIGKILL exit without inferring its cause", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGTERM");
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -224,49 +207,24 @@ it("records a scripted SIGKILL exit without inferring its cause", async () => {
|
||||
expect(exit.signal, "the observed SIGKILL remains an exit fact").toBe(9);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[3, 3],
|
||||
[0, 0],
|
||||
[7, 7],
|
||||
] as const)(
|
||||
"records exit code %i of a finalizer that exits on its own as %i",
|
||||
async (exitCode, expected) => {
|
||||
it.each([0, 3])(
|
||||
"records exit code %i of a finalizer that exits on its own",
|
||||
async (exitCode) => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGTERM");
|
||||
fake.deliverExit({ exitCode, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.exit_code,
|
||||
"the PTY-reported code of an exit not caused by a signal is recorded",
|
||||
).toBe(expected);
|
||||
expect(exit.reason, "the initiating deadline stays the exit reason").toBe(
|
||||
"timeout",
|
||||
);
|
||||
expect(exit.finalization?.exit_code).toBe(exitCode);
|
||||
expect(exit.reason).toBe("timeout");
|
||||
},
|
||||
);
|
||||
|
||||
it("records no exit code when the finalizing target was ended by a signal", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.exit_code,
|
||||
"a code reported next to a terminating signal is never recorded",
|
||||
).toBeNull();
|
||||
expect(exit.signal, "the signal remains the observed exit fact").toBe(9);
|
||||
});
|
||||
|
||||
it("records no exit code when the PTY reports a negative code without a signal", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGTERM");
|
||||
fake.deliverExit({ exitCode: -1, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -280,24 +238,11 @@ it("records no exit code when the PTY reports a negative code without a signal",
|
||||
);
|
||||
});
|
||||
|
||||
it("records no exit code when no exit was observed", async () => {
|
||||
const fake = fakeTerminal({}, new Promise<void>(() => undefined));
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(result.unobserved, "no exit is invented").toBe(true);
|
||||
expect(
|
||||
result.finalization?.exit_code,
|
||||
"an unobserved exit has no code to record",
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("records every delivery result and publishes attempts incrementally", async () => {
|
||||
snapshots.length = 0;
|
||||
const fake = fakeTerminal({ SIGTERM: "gone", SIGKILL: "gone" });
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGKILL");
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -347,6 +292,7 @@ it.each(["identity-changed", "unverified"] as const)(
|
||||
);
|
||||
|
||||
it("returns at once when a cancellation kill cannot be delivered", async () => {
|
||||
fakePollingClock();
|
||||
const fake = fakeTerminal({ SIGKILL: "unverified" });
|
||||
const controller = new AbortController();
|
||||
const pending = start(
|
||||
@@ -354,8 +300,11 @@ it("returns at once when a cancellation kill cannot be delivered", async () => {
|
||||
{ timeout_ms: 100, finalization_ms: 60_000 },
|
||||
controller.signal,
|
||||
);
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(fake.signals).toEqual(["SIGTERM"]);
|
||||
controller.abort();
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(fake.signals).toEqual(["SIGTERM", "SIGKILL"]);
|
||||
|
||||
const result = await pending;
|
||||
|
||||
@@ -370,16 +319,21 @@ it("waits for a pending delivery before reporting an observed exit", async () =>
|
||||
});
|
||||
const fake = fakeTerminal({ SIGTERM: "signaled" }, hold);
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
let settled = false;
|
||||
void pending.then(() => {
|
||||
settled = true;
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
expect(settled, "the result waits for the outstanding delivery").toBe(false);
|
||||
|
||||
release();
|
||||
try {
|
||||
await expect.poll(() => fake.signals).toContain("SIGTERM");
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
const result = await Promise.race([
|
||||
pending,
|
||||
new Promise<void>((resolve) => setImmediate(resolve)),
|
||||
]);
|
||||
expect(
|
||||
result,
|
||||
"the result waits for the outstanding delivery",
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
release();
|
||||
await Promise.allSettled([pending]);
|
||||
}
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
@@ -388,25 +342,10 @@ it("waits for a pending delivery before reporting an observed exit", async () =>
|
||||
).toMatchObject([{ signal: "SIGTERM", delivery: "signaled" }]);
|
||||
});
|
||||
|
||||
it("keeps the zero-interval deadline as a bare SIGKILL", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 60, finalization_ms: 0 });
|
||||
await waitFor(() => fake.signals.includes("terminal:SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
fake.signals.filter((signal) => !signal.startsWith("terminal:")),
|
||||
"the identity-checked signaller is never used without an interval",
|
||||
).toEqual([]);
|
||||
expect(exit.finalization, "no finalization record exists").toBeUndefined();
|
||||
});
|
||||
|
||||
it("stops waiting for a hung delivery once the exit is observed", async () => {
|
||||
const fake = fakeTerminal({}, new Promise<void>(() => undefined));
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGTERM");
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -431,7 +370,7 @@ it("retains a thrown signal error as an unverified delivery reason", async () =>
|
||||
new Error("signal transport failed"),
|
||||
);
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGTERM");
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
@@ -455,7 +394,7 @@ it("keeps an observed exit when a late kill delivery reports unverified", async
|
||||
});
|
||||
const fake = fakeTerminal({ SIGKILL: "unverified" }, hold);
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGKILL");
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
release();
|
||||
|
||||
@@ -472,15 +411,30 @@ it("keeps an observed exit when a late kill delivery reports unverified", async
|
||||
|
||||
it("stops waiting when the escalation delivery never settles and no exit arrives", async () => {
|
||||
const fake = fakeTerminal({}, new Promise<void>(() => undefined));
|
||||
const waitStarted = Date.now();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
const escalation = phaseArrival();
|
||||
const pending = start(
|
||||
{
|
||||
...fake,
|
||||
signalTarget: (signal) => {
|
||||
if (signal === "SIGKILL") escalation.arrive();
|
||||
return fake.signalTarget(signal);
|
||||
},
|
||||
},
|
||||
{ timeout_ms: 100, finalization_ms: 100 },
|
||||
);
|
||||
await escalation.reached(pending, "SIGKILL delivery");
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(
|
||||
Date.now() - waitStarted,
|
||||
"the wait is bounded by the one-second cleanup grace after the interval",
|
||||
).toBeLessThan(2_500);
|
||||
// The production grace timer is already scheduled. Race it against the
|
||||
// independent test bound so time spent reaching escalation cannot fail the test.
|
||||
const watchdog = new AbortController();
|
||||
const result = await Promise.race([
|
||||
pending,
|
||||
delay(1_500, null, { signal: watchdog.signal }),
|
||||
]).finally(() => watchdog.abort());
|
||||
if (result === null) {
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
throw new Error("Hung SIGKILL delivery outlived its cleanup grace");
|
||||
}
|
||||
expect(result.unobserved, "no exit is invented").toBe(true);
|
||||
expect(
|
||||
result.finalization?.signals.at(-1),
|
||||
@@ -490,32 +444,20 @@ it("stops waiting when the escalation delivery never settles and no exit arrives
|
||||
delivery: "unverified",
|
||||
reason: "delivery did not settle within 1000 ms",
|
||||
});
|
||||
expect(result.finalization?.elapsed_ms, "no exit was observed").toBeNull();
|
||||
expect(result.finalization).toMatchObject({
|
||||
elapsed_ms: null,
|
||||
exit_code: null,
|
||||
});
|
||||
});
|
||||
|
||||
const activeTimers = (): number =>
|
||||
process.getActiveResourcesInfo().filter((name) => name === "Timeout").length;
|
||||
|
||||
it("leaves no referenced timer behind after a zero-interval exit", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const before = activeTimers();
|
||||
const pending = start(fake, { timeout_ms: 60, finalization_ms: 0 });
|
||||
await waitFor(() => fake.signals.includes("terminal:SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
await pending;
|
||||
|
||||
expect(
|
||||
activeTimers(),
|
||||
"the zero path resolves without a bounded-wait timer",
|
||||
).toBeLessThanOrEqual(before);
|
||||
});
|
||||
|
||||
it("cancels the bounded-wait timer once deliveries settle after an exit", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const before = activeTimers();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
await expect.poll(() => fake.signals).toContain("SIGTERM");
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
await pending;
|
||||
@@ -538,7 +480,7 @@ it("keeps a non-empty reason when a signal delivery throws without a message", a
|
||||
},
|
||||
};
|
||||
const pending = start(throwing, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => signalStarted);
|
||||
await expect.poll(() => signalStarted).toBe(true);
|
||||
throwing.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
Reference in new issue
Block a user