test(web): consolidate scenario Evidence delivery checks (#13161)

This commit is contained in:
morluto authored and GitHub committed 2026-11-21 17:25:33 +00:00
1 parent b0043f1fe4
commit 8d17fcb7a2
2 files changed
+152 -186

No files matched your search

@@ -13,17 +13,57 @@ import { PlaywrightBrowserScenarioProvider } from "../../../src/browser/Playwrig
import { scenarioCaptureFixture } from "../../fixtures/browserScenarioCapture.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import type { BrowserScenarioCapturePort } from "../../../src/ports/BrowserScenarioCapturePort.js";
import { controlledCleanup } from "../../support/lifecycleHarness.js";
const identity = { id: "fixture", name: "Fixture", version: "1" };
const capture = scenarioCaptureFixture();
const runCli = async (
provider: BrowserScenarioCapturePort,
options: readonly string[] = [],
) => {
const cli = createCommandCli();
registerBrowserScenarioCommands(
cli,
pino({ level: "silent" }),
{},
{
createProvider: () => provider,
},
);
let stdout = "";
await cli.serve(
[
"capture-browser-scenario",
JSON.stringify({
browser: { mode: "launch", executable_path: process.execPath },
start_url: { url: "http://127.0.0.1:3101/" },
actions: [
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
],
}),
...options,
"--json",
],
{
env: {},
stdout: (text) => {
stdout += text;
},
exit: () => undefined,
},
);
return JSON.parse(stdout) as unknown;
};
it.each([false, true])(
"retains startup failure and reports failed cleanup retry when retryFails=%s",
"keeps the initial startup failure snapshot when cleanup refusal is %s",
async (retryFails) => {
let refuseCleanup = retryFails;
let cleanupAttempts = 0;
const lease = new AnalysisCleanupLease(async () => {
cleanupAttempts += 1;
if (refuseCleanup) throw new Error("profile removal denied");
}, ["owned-profile"]);
// The production provider owns resources retained by a partially failed opener.
// No browser is launched by this boundary regression.
const cleanup = controlledCleanup({
refused: retryFails,
failure: new Error("profile removal denied"),
});
const lease = new AnalysisCleanupLease(cleanup.close, ["owned-profile"]);
const provider = new PlaywrightBrowserScenarioProvider(
async (_scenario, options) => {
options.retainCleanup?.(lease);
@@ -33,129 +73,77 @@ it.each([false, true])(
);
},
);
const cli = createCommandCli();
let stdout = "";
registerBrowserScenarioCommands(
cli,
pino({ level: "silent" }),
{},
{
createProvider: () => provider,
},
);
await cli.serve(
[
"capture-browser-scenario",
JSON.stringify({
browser: { mode: "launch", executable_path: process.execPath },
start_url: { url: "http://127.0.0.1:3101/" },
actions: [
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
],
}),
"--json",
],
{
env: {},
stdout: (text) => {
stdout += text;
},
exit: () => undefined,
},
);
const failure = JSON.parse(stdout);
expect(failure).toMatchObject({
code: "cleanup_incomplete",
details: { reason: "browser startup failed" },
});
if (retryFails) {
try {
const failure = await runCli(provider);
expect(failure).toMatchObject({
details: {
diagnostics: {
primary_error: {
code: "cleanup_incomplete",
details: { reason: "browser startup failed" },
},
cleanup_error: {
details: { diagnostics: { reason: "profile removal denied" } },
code: "cleanup_incomplete",
details: { reason: "browser startup failed" },
});
expect(cleanup.attempts).toBe(1);
expect(lease.released).toBe(!retryFails);
if (retryFails) {
expect(failure).toMatchObject({
details: {
diagnostics: {
primary_error: {
code: "cleanup_incomplete",
details: { reason: "browser startup failed" },
},
cleanup_error: {
details: { diagnostics: { reason: "profile removal denied" } },
},
},
},
},
});
expect(lease.released).toBe(false);
refuseCleanup = false;
});
cleanup.allow();
await provider.close();
expect(cleanup.attempts).toBe(2);
} else
expect(JSON.stringify(failure)).not.toContain("profile removal denied");
expect(lease.released).toBe(true);
} finally {
cleanup.allow();
await provider.close();
expect(cleanupAttempts).toBe(2);
} else {
expect(JSON.stringify(failure)).not.toContain("profile removal denied");
expect(cleanupAttempts).toBe(1);
}
expect(lease.released).toBe(true);
},
);
it.each(["complete", "summary"] as const)(
"retains complete Evidence on cancellation during CLI cleanup with %s delivery",
async (detail) => {
const root = await createTestTempDirectory("rea-browser-summary-cli-");
const destination = join(root, "evidence.json");
const destination = join(
await createTestTempDirectory("rea-browser-summary-cli-"),
"evidence.json",
);
const originalExit = process.exitCode;
const listeners = new Set(process.listeners("SIGINT"));
const cli = createCommandCli();
let stdout = "";
let closed = false;
registerBrowserScenarioCommands(
cli,
pino({ level: "silent" }),
{},
{
createProvider: () => ({
identity: () => ({ id: "fixture", name: "Fixture", version: "1" }),
captureScenario: async () => ok(scenarioCaptureFixture()),
close: async () => {
closed = true;
// Invoke only this command's newly installed listener, without emitting
// a process-wide signal into the shared test worker.
const installed = process
.listeners("SIGINT")
.filter((listener) => !listeners.has(listener));
expect(installed).toHaveLength(1);
installed[0]!("SIGINT");
},
}),
const provider: BrowserScenarioCapturePort = {
identity: () => identity,
captureScenario: async () => ok(capture),
close: async () => {
closed = true;
// Invoke only this command's newly installed listener in the shared worker.
const installed = process
.listeners("SIGINT")
.filter((listener) => !listeners.has(listener));
expect(installed).toHaveLength(1);
installed[0]!("SIGINT");
},
);
};
try {
await cli.serve(
[
"capture-browser-scenario",
JSON.stringify({
browser: { mode: "launch", executable_path: process.execPath },
start_url: { url: "http://127.0.0.1:3101/" },
actions: [
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
],
}),
"--detail",
detail,
"--evidence_output",
destination,
"--json",
],
{
env: {},
stdout: (text) => {
stdout += text;
},
exit: () => undefined,
},
);
expect(JSON.parse(stdout)).toMatchObject({
const output = await runCli(provider, [
"--detail",
detail,
"--evidence_output",
destination,
]);
expect(output).toMatchObject({
code: "cancelled",
details: {
partial_observation: {
operation: "capture_browser_scenario",
normalized_result: { steps: expect.any(Array) },
normalized_result: capture,
},
},
});
@@ -171,59 +159,37 @@ it.each(["complete", "summary"] as const)(
);
it("keeps the provider's cleanup failure and completed Evidence without publishing a summary bundle", async () => {
const root = await createTestTempDirectory("rea-browser-summary-cleanup-");
const destination = join(root, "evidence.json");
const cli = createCommandCli();
let stdout = "";
const destination = join(
await createTestTempDirectory("rea-browser-summary-cleanup-"),
"evidence.json",
);
const failure = new ProviderCleanupError(
"owned-browser",
["owned-profile"],
{ reason: "cleanup denied" },
{ operation: "capture_browser_scenario" },
);
registerBrowserScenarioCommands(
cli,
pino({ level: "silent" }),
{},
{
createProvider: () => ({
identity: () => ({ id: "fixture", name: "Fixture", version: "1" }),
captureScenario: async () => ok(scenarioCaptureFixture()),
close: async () => {
throw failure;
},
}),
const provider: BrowserScenarioCapturePort = {
identity: () => identity,
captureScenario: async () => ok(capture),
close: async () => {
throw failure;
},
);
await cli.serve(
[
"capture-browser-scenario",
JSON.stringify({
browser: { mode: "launch", executable_path: process.execPath },
start_url: { url: "http://127.0.0.1:3101/" },
actions: [
{ step_id: "browse", action: "wait_for_timeout", duration_ms: 1 },
],
}),
"--detail",
"summary",
"--evidence_output",
destination,
"--json",
],
{
env: {},
stdout: (text) => {
stdout += text;
},
exit: () => undefined,
},
);
expect(JSON.parse(stdout)).toMatchObject({
};
const output = await runCli(provider, [
"--detail",
"summary",
"--evidence_output",
destination,
]);
expect(output).toMatchObject({
code: "cleanup_incomplete",
details: {
resources: ["owned-profile"],
partial_observation: { operation: "capture_browser_scenario" },
partial_observation: {
operation: "capture_browser_scenario",
normalized_result: capture,
},
},
});
await expect(access(destination)).rejects.toMatchObject({ code: "ENOENT" });
+31 -31
View File
@@ -354,6 +354,28 @@ describe("browser scenario MCP tool", () => {
});
});
const openScenarioMcp = async (
provider: BrowserScenarioCapturePort,
name: string,
) => {
const session = createTestBinarySession(() => ({
execute: () => Promise.resolve(observed(null)),
close: () => Promise.resolve(ok(null)),
}));
const { client } = await createApplicationMcpHarness({
session,
clientIdentity: { name, version: "1" },
serverOptions: {
browserScenarioCapture: provider,
availabilityPolicy: () => ({
processCaptureEnabled: false,
investigationInputRoots: 0,
}),
},
});
return { client, session };
};
test("records a completed late-close cancellation as server Evidence", async () => {
const provider: BrowserScenarioCapturePort = {
identity: () => providerIdentity,
@@ -366,21 +388,10 @@ test("records a completed late-close cancellation as server Evidence", async ()
);
},
};
const session = createTestBinarySession(() => ({
execute: () => Promise.resolve(observed(null)),
close: () => Promise.resolve(ok(null)),
}));
const { client } = await createApplicationMcpHarness({
session,
clientIdentity: { name: "scenario-late-close", version: "1" },
serverOptions: {
browserScenarioCapture: provider,
availabilityPolicy: () => ({
processCaptureEnabled: false,
investigationInputRoots: 0,
}),
},
});
const { client, session } = await openScenarioMcp(
provider,
"scenario-late-close",
);
const response = await client.callTool({
name: "capture_browser_scenario",
arguments: minimalScenario(),
@@ -411,23 +422,12 @@ test("records a completed late-close cancellation as server Evidence", async ()
await expect(client.ping()).resolves.toEqual({});
});
test("delivers a retained summary and pages its exact steps and events", async () => {
test("delivers a retained summary and pages its exact steps", async () => {
const provider = new FakeBrowserScenarioProvider();
const session = createTestBinarySession(() => ({
execute: () => Promise.resolve(observed(null)),
close: () => Promise.resolve(ok(null)),
}));
const { client } = await createApplicationMcpHarness({
session,
clientIdentity: { name: "scenario-summary", version: "1" },
serverOptions: {
browserScenarioCapture: provider,
availabilityPolicy: () => ({
processCaptureEnabled: false,
investigationInputRoots: 0,
}),
},
});
const { client, session } = await openScenarioMcp(
provider,
"scenario-summary",
);
const tools = (await client.listTools()).tools;
const advertised = tools.find(
({ name }) => name === "capture_browser_scenario",