mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 22:15:50 +00:00
test(web): consolidate scenario Evidence delivery checks (#13161)
This commit is contained in:
1 parent
b0043f1fe4
commit
8d17fcb7a2
2 files changed
+152
-186
No files matched your search
@@ -13,17 +13,57 @@ import { PlaywrightBrowserScenarioProvider } from "../../../src/browser/Playwrig
|
||||
import { scenarioCaptureFixture } from "../../fixtures/browserScenarioCapture.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
import type { BrowserScenarioCapturePort } from "../../../src/ports/BrowserScenarioCapturePort.js";
|
||||
import { controlledCleanup } from "../../support/lifecycleHarness.js";
|
||||
|
||||
const identity = { id: "fixture", name: "Fixture", version: "1" };
|
||||
const capture = scenarioCaptureFixture();
|
||||
const runCli = async (
|
||||
provider: BrowserScenarioCapturePort,
|
||||
options: readonly string[] = [],
|
||||
) => {
|
||||
const cli = createCommandCli();
|
||||
registerBrowserScenarioCommands(
|
||||
cli,
|
||||
pino({ level: "silent" }),
|
||||
{},
|
||||
{
|
||||
createProvider: () => provider,
|
||||
},
|
||||
);
|
||||
let stdout = "";
|
||||
await cli.serve(
|
||||
[
|
||||
"capture-browser-scenario",
|
||||
JSON.stringify({
|
||||
browser: { mode: "launch", executable_path: process.execPath },
|
||||
start_url: { url: "http://127.0.0.1:3101/" },
|
||||
actions: [
|
||||
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
|
||||
],
|
||||
}),
|
||||
...options,
|
||||
"--json",
|
||||
],
|
||||
{
|
||||
env: {},
|
||||
stdout: (text) => {
|
||||
stdout += text;
|
||||
},
|
||||
exit: () => undefined,
|
||||
},
|
||||
);
|
||||
return JSON.parse(stdout) as unknown;
|
||||
};
|
||||
|
||||
it.each([false, true])(
|
||||
"retains startup failure and reports failed cleanup retry when retryFails=%s",
|
||||
"keeps the initial startup failure snapshot when cleanup refusal is %s",
|
||||
async (retryFails) => {
|
||||
let refuseCleanup = retryFails;
|
||||
let cleanupAttempts = 0;
|
||||
const lease = new AnalysisCleanupLease(async () => {
|
||||
cleanupAttempts += 1;
|
||||
if (refuseCleanup) throw new Error("profile removal denied");
|
||||
}, ["owned-profile"]);
|
||||
// The production provider owns resources retained by a partially failed opener.
|
||||
// No browser is launched by this boundary regression.
|
||||
const cleanup = controlledCleanup({
|
||||
refused: retryFails,
|
||||
failure: new Error("profile removal denied"),
|
||||
});
|
||||
const lease = new AnalysisCleanupLease(cleanup.close, ["owned-profile"]);
|
||||
const provider = new PlaywrightBrowserScenarioProvider(
|
||||
async (_scenario, options) => {
|
||||
options.retainCleanup?.(lease);
|
||||
@@ -33,129 +73,77 @@ it.each([false, true])(
|
||||
);
|
||||
},
|
||||
);
|
||||
const cli = createCommandCli();
|
||||
let stdout = "";
|
||||
registerBrowserScenarioCommands(
|
||||
cli,
|
||||
pino({ level: "silent" }),
|
||||
{},
|
||||
{
|
||||
createProvider: () => provider,
|
||||
},
|
||||
);
|
||||
await cli.serve(
|
||||
[
|
||||
"capture-browser-scenario",
|
||||
JSON.stringify({
|
||||
browser: { mode: "launch", executable_path: process.execPath },
|
||||
start_url: { url: "http://127.0.0.1:3101/" },
|
||||
actions: [
|
||||
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
|
||||
],
|
||||
}),
|
||||
"--json",
|
||||
],
|
||||
{
|
||||
env: {},
|
||||
stdout: (text) => {
|
||||
stdout += text;
|
||||
},
|
||||
exit: () => undefined,
|
||||
},
|
||||
);
|
||||
const failure = JSON.parse(stdout);
|
||||
expect(failure).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: { reason: "browser startup failed" },
|
||||
});
|
||||
if (retryFails) {
|
||||
try {
|
||||
const failure = await runCli(provider);
|
||||
expect(failure).toMatchObject({
|
||||
details: {
|
||||
diagnostics: {
|
||||
primary_error: {
|
||||
code: "cleanup_incomplete",
|
||||
details: { reason: "browser startup failed" },
|
||||
},
|
||||
cleanup_error: {
|
||||
details: { diagnostics: { reason: "profile removal denied" } },
|
||||
code: "cleanup_incomplete",
|
||||
details: { reason: "browser startup failed" },
|
||||
});
|
||||
expect(cleanup.attempts).toBe(1);
|
||||
expect(lease.released).toBe(!retryFails);
|
||||
if (retryFails) {
|
||||
expect(failure).toMatchObject({
|
||||
details: {
|
||||
diagnostics: {
|
||||
primary_error: {
|
||||
code: "cleanup_incomplete",
|
||||
details: { reason: "browser startup failed" },
|
||||
},
|
||||
cleanup_error: {
|
||||
details: { diagnostics: { reason: "profile removal denied" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(lease.released).toBe(false);
|
||||
refuseCleanup = false;
|
||||
});
|
||||
cleanup.allow();
|
||||
await provider.close();
|
||||
expect(cleanup.attempts).toBe(2);
|
||||
} else
|
||||
expect(JSON.stringify(failure)).not.toContain("profile removal denied");
|
||||
expect(lease.released).toBe(true);
|
||||
} finally {
|
||||
cleanup.allow();
|
||||
await provider.close();
|
||||
expect(cleanupAttempts).toBe(2);
|
||||
} else {
|
||||
expect(JSON.stringify(failure)).not.toContain("profile removal denied");
|
||||
expect(cleanupAttempts).toBe(1);
|
||||
}
|
||||
expect(lease.released).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["complete", "summary"] as const)(
|
||||
"retains complete Evidence on cancellation during CLI cleanup with %s delivery",
|
||||
async (detail) => {
|
||||
const root = await createTestTempDirectory("rea-browser-summary-cli-");
|
||||
const destination = join(root, "evidence.json");
|
||||
const destination = join(
|
||||
await createTestTempDirectory("rea-browser-summary-cli-"),
|
||||
"evidence.json",
|
||||
);
|
||||
const originalExit = process.exitCode;
|
||||
const listeners = new Set(process.listeners("SIGINT"));
|
||||
const cli = createCommandCli();
|
||||
let stdout = "";
|
||||
let closed = false;
|
||||
registerBrowserScenarioCommands(
|
||||
cli,
|
||||
pino({ level: "silent" }),
|
||||
{},
|
||||
{
|
||||
createProvider: () => ({
|
||||
identity: () => ({ id: "fixture", name: "Fixture", version: "1" }),
|
||||
captureScenario: async () => ok(scenarioCaptureFixture()),
|
||||
close: async () => {
|
||||
closed = true;
|
||||
// Invoke only this command's newly installed listener, without emitting
|
||||
// a process-wide signal into the shared test worker.
|
||||
const installed = process
|
||||
.listeners("SIGINT")
|
||||
.filter((listener) => !listeners.has(listener));
|
||||
expect(installed).toHaveLength(1);
|
||||
installed[0]!("SIGINT");
|
||||
},
|
||||
}),
|
||||
const provider: BrowserScenarioCapturePort = {
|
||||
identity: () => identity,
|
||||
captureScenario: async () => ok(capture),
|
||||
close: async () => {
|
||||
closed = true;
|
||||
// Invoke only this command's newly installed listener in the shared worker.
|
||||
const installed = process
|
||||
.listeners("SIGINT")
|
||||
.filter((listener) => !listeners.has(listener));
|
||||
expect(installed).toHaveLength(1);
|
||||
installed[0]!("SIGINT");
|
||||
},
|
||||
);
|
||||
};
|
||||
try {
|
||||
await cli.serve(
|
||||
[
|
||||
"capture-browser-scenario",
|
||||
JSON.stringify({
|
||||
browser: { mode: "launch", executable_path: process.execPath },
|
||||
start_url: { url: "http://127.0.0.1:3101/" },
|
||||
actions: [
|
||||
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
|
||||
],
|
||||
}),
|
||||
"--detail",
|
||||
detail,
|
||||
"--evidence_output",
|
||||
destination,
|
||||
"--json",
|
||||
],
|
||||
{
|
||||
env: {},
|
||||
stdout: (text) => {
|
||||
stdout += text;
|
||||
},
|
||||
exit: () => undefined,
|
||||
},
|
||||
);
|
||||
expect(JSON.parse(stdout)).toMatchObject({
|
||||
const output = await runCli(provider, [
|
||||
"--detail",
|
||||
detail,
|
||||
"--evidence_output",
|
||||
destination,
|
||||
]);
|
||||
expect(output).toMatchObject({
|
||||
code: "cancelled",
|
||||
details: {
|
||||
partial_observation: {
|
||||
operation: "capture_browser_scenario",
|
||||
normalized_result: { steps: expect.any(Array) },
|
||||
normalized_result: capture,
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -171,59 +159,37 @@ it.each(["complete", "summary"] as const)(
|
||||
);
|
||||
|
||||
it("keeps the provider's cleanup failure and completed Evidence without publishing a summary bundle", async () => {
|
||||
const root = await createTestTempDirectory("rea-browser-summary-cleanup-");
|
||||
const destination = join(root, "evidence.json");
|
||||
const cli = createCommandCli();
|
||||
let stdout = "";
|
||||
const destination = join(
|
||||
await createTestTempDirectory("rea-browser-summary-cleanup-"),
|
||||
"evidence.json",
|
||||
);
|
||||
const failure = new ProviderCleanupError(
|
||||
"owned-browser",
|
||||
["owned-profile"],
|
||||
{ reason: "cleanup denied" },
|
||||
{ operation: "capture_browser_scenario" },
|
||||
);
|
||||
registerBrowserScenarioCommands(
|
||||
cli,
|
||||
pino({ level: "silent" }),
|
||||
{},
|
||||
{
|
||||
createProvider: () => ({
|
||||
identity: () => ({ id: "fixture", name: "Fixture", version: "1" }),
|
||||
captureScenario: async () => ok(scenarioCaptureFixture()),
|
||||
close: async () => {
|
||||
throw failure;
|
||||
},
|
||||
}),
|
||||
const provider: BrowserScenarioCapturePort = {
|
||||
identity: () => identity,
|
||||
captureScenario: async () => ok(capture),
|
||||
close: async () => {
|
||||
throw failure;
|
||||
},
|
||||
);
|
||||
await cli.serve(
|
||||
[
|
||||
"capture-browser-scenario",
|
||||
JSON.stringify({
|
||||
browser: { mode: "launch", executable_path: process.execPath },
|
||||
start_url: { url: "http://127.0.0.1:3101/" },
|
||||
actions: [
|
||||
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
|
||||
],
|
||||
}),
|
||||
"--detail",
|
||||
"summary",
|
||||
"--evidence_output",
|
||||
destination,
|
||||
"--json",
|
||||
],
|
||||
{
|
||||
env: {},
|
||||
stdout: (text) => {
|
||||
stdout += text;
|
||||
},
|
||||
exit: () => undefined,
|
||||
},
|
||||
);
|
||||
expect(JSON.parse(stdout)).toMatchObject({
|
||||
};
|
||||
const output = await runCli(provider, [
|
||||
"--detail",
|
||||
"summary",
|
||||
"--evidence_output",
|
||||
destination,
|
||||
]);
|
||||
expect(output).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
resources: ["owned-profile"],
|
||||
partial_observation: { operation: "capture_browser_scenario" },
|
||||
partial_observation: {
|
||||
operation: "capture_browser_scenario",
|
||||
normalized_result: capture,
|
||||
},
|
||||
},
|
||||
});
|
||||
await expect(access(destination)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
|
||||
@@ -354,6 +354,28 @@ describe("browser scenario MCP tool", () => {
|
||||
});
|
||||
});
|
||||
|
||||
const openScenarioMcp = async (
|
||||
provider: BrowserScenarioCapturePort,
|
||||
name: string,
|
||||
) => {
|
||||
const session = createTestBinarySession(() => ({
|
||||
execute: () => Promise.resolve(observed(null)),
|
||||
close: () => Promise.resolve(ok(null)),
|
||||
}));
|
||||
const { client } = await createApplicationMcpHarness({
|
||||
session,
|
||||
clientIdentity: { name, version: "1" },
|
||||
serverOptions: {
|
||||
browserScenarioCapture: provider,
|
||||
availabilityPolicy: () => ({
|
||||
processCaptureEnabled: false,
|
||||
investigationInputRoots: 0,
|
||||
}),
|
||||
},
|
||||
});
|
||||
return { client, session };
|
||||
};
|
||||
|
||||
test("records a completed late-close cancellation as server Evidence", async () => {
|
||||
const provider: BrowserScenarioCapturePort = {
|
||||
identity: () => providerIdentity,
|
||||
@@ -366,21 +388,10 @@ test("records a completed late-close cancellation as server Evidence", async ()
|
||||
);
|
||||
},
|
||||
};
|
||||
const session = createTestBinarySession(() => ({
|
||||
execute: () => Promise.resolve(observed(null)),
|
||||
close: () => Promise.resolve(ok(null)),
|
||||
}));
|
||||
const { client } = await createApplicationMcpHarness({
|
||||
session,
|
||||
clientIdentity: { name: "scenario-late-close", version: "1" },
|
||||
serverOptions: {
|
||||
browserScenarioCapture: provider,
|
||||
availabilityPolicy: () => ({
|
||||
processCaptureEnabled: false,
|
||||
investigationInputRoots: 0,
|
||||
}),
|
||||
},
|
||||
});
|
||||
const { client, session } = await openScenarioMcp(
|
||||
provider,
|
||||
"scenario-late-close",
|
||||
);
|
||||
const response = await client.callTool({
|
||||
name: "capture_browser_scenario",
|
||||
arguments: minimalScenario(),
|
||||
@@ -411,23 +422,12 @@ test("records a completed late-close cancellation as server Evidence", async ()
|
||||
await expect(client.ping()).resolves.toEqual({});
|
||||
});
|
||||
|
||||
test("delivers a retained summary and pages its exact steps and events", async () => {
|
||||
test("delivers a retained summary and pages its exact steps", async () => {
|
||||
const provider = new FakeBrowserScenarioProvider();
|
||||
const session = createTestBinarySession(() => ({
|
||||
execute: () => Promise.resolve(observed(null)),
|
||||
close: () => Promise.resolve(ok(null)),
|
||||
}));
|
||||
const { client } = await createApplicationMcpHarness({
|
||||
session,
|
||||
clientIdentity: { name: "scenario-summary", version: "1" },
|
||||
serverOptions: {
|
||||
browserScenarioCapture: provider,
|
||||
availabilityPolicy: () => ({
|
||||
processCaptureEnabled: false,
|
||||
investigationInputRoots: 0,
|
||||
}),
|
||||
},
|
||||
});
|
||||
const { client, session } = await openScenarioMcp(
|
||||
provider,
|
||||
"scenario-summary",
|
||||
);
|
||||
const tools = (await client.listTools()).tools;
|
||||
const advertised = tools.find(
|
||||
({ name }) => name === "capture_browser_scenario",
|
||||
|
||||
Reference in new issue
Block a user