test(web): consolidate scenario input diagnostics (#13159)

This commit is contained in:
morluto authored and GitHub committed 2026-11-21 12:20:56 -05:00
1 parent 52437cd431
commit 0fa220c735
3 files changed
+47 -116

No files matched your search

-25
View File
@@ -2,7 +2,6 @@ import { createHash } from "node:crypto";
import { describe, expect, it } from "vitest";
import { BrowserScenarioSecrets } from "./BrowserScenarioSecrets.js";
import { resolveScenarioSecrets } from "../../tests/fixtures/browserScenarioSecrets.js";
import { browserScenarioSchema } from "../domain/browserScenario.js";
import { DEFAULT_BROWSER_SCENARIO_CAPTURE_LIMITS } from "../domain/browserScenarioCaptureLimits.js";
@@ -125,30 +124,6 @@ describe("browser scenario secret redaction", () => {
value_sha256: null,
});
});
it("does not resolve inherited environment properties as secret values", () => {
const scenario = browserScenarioSchema.parse(baseScenario());
const environment = Object.assign(
Object.create({ REA_TEST_SESSION: "session" }) as Record<string, string>,
{ REA_TEST_PASSWORD: "password", REA_TEST_COOKIE: "cookie" },
);
expect(BrowserScenarioSecrets.resolve(scenario, environment)).toMatchObject(
{
ok: false,
error: {
_tag: "AnalysisInputError",
issues: [
{
path: ["secrets", 0, "environment_variable"],
expected: {
environment_variable: "REA_TEST_SESSION",
secret_id: "session",
},
},
],
},
},
);
});
});
describe("storage value fingerprint encoding", () => {
@@ -53,15 +53,6 @@ const launchScenario = {
actions: [{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 }],
} as const;
it("accepts a valid scenario without launching or attaching to a browser", async () => {
const output = await serveJson(
registerValidatingCli(),
"validate-browser-scenario",
launchScenario,
);
expect(output).toEqual({ valid: true, actions: 1, secrets: 0 });
});
it.each(["validate-browser-scenario", "capture-browser-scenario"])(
"%s rejects a literal Basic username containing a colon before creating a provider",
async (command) => {
@@ -104,53 +95,40 @@ it.each(["validate-browser-scenario", "capture-browser-scenario"])(
},
);
it("counts declared secrets without resolving their environment values", async () => {
const cli = registerValidatingCli();
const output = await serveJson(cli, "validate-browser-scenario", {
...launchScenario,
actions: [
{
step_id: "sign-in",
action: "fill",
locator: { kind: "css", selector: "#token" },
value: { source: "secret", secret_id: "session_token" },
},
],
secrets: [
{ secret_id: "session_token", environment_variable: "SESSION_TOKEN" },
{ secret_id: "api_key", environment_variable: "API_KEY" },
],
});
expect(output).toEqual({ valid: true, actions: 1, secrets: 2 });
});
const secretAction = {
step_id: "sign-in",
action: "fill",
locator: { kind: "css", selector: "#token" },
value: { source: "secret", secret_id: "session_token" },
};
const disconnectedBrowser = {
mode: "connect",
cdp_endpoint: "http://127.0.0.1:1",
target_id: "unavailable-target",
};
it("rejects a duplicate action step identity with the offending field path", async () => {
const output = await serveJson(
registerValidatingCli(),
"validate-browser-scenario",
{
...launchScenario,
actions: [
{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 },
{ step_id: "settle", action: "wait_for_timeout", duration_ms: 2 },
],
},
);
expect(output).toMatchObject({
error: "Browser scenario validation failed",
code: "invalid_request",
category: "invalid_input",
details: {
issues: [
expect.objectContaining({
path: ["actions"],
reason: "invalid_value",
message: "Action step IDs must be unique",
}),
],
},
});
});
it.each([
["launch", launchScenario.browser],
["connect", disconnectedBrowser],
])(
"validates %s and counts secrets without touching runtime resources",
async (_mode, browser) => {
const output = await serveJson(
registerValidatingCli(),
"validate-browser-scenario",
{
...launchScenario,
browser,
actions: [secretAction],
secrets: [
{ secret_id: "session_token", environment_variable: "SESSION_TOKEN" },
{ secret_id: "api_key", environment_variable: "API_KEY" },
],
},
);
expect(output).toEqual({ valid: true, actions: 1, secrets: 2 });
},
);
it("rejects an action that reuses the reserved initial step identity", async () => {
const output = await serveJson(
@@ -186,14 +164,7 @@ it("rejects an undeclared secret reference without exposing any secret value", a
const cli = registerValidatingCli();
const output = await serveJson(cli, "validate-browser-scenario", {
...launchScenario,
actions: [
{
step_id: "sign-in",
action: "fill",
locator: { kind: "css", selector: "#token" },
value: { source: "secret", secret_id: "session_token" },
},
],
actions: [secretAction],
});
const serialized = JSON.stringify(output);
expect(output).toMatchObject({
@@ -240,19 +211,6 @@ it("rejects an unknown field instead of ignoring it", async () => {
});
});
it("accepts a disconnected CDP target without constructing a provider", async () => {
expect(
await serveJson(registerValidatingCli(), "validate-browser-scenario", {
...launchScenario,
browser: {
mode: "connect",
cdp_endpoint: "http://127.0.0.1:1",
target_id: "unavailable-target",
},
}),
).toEqual({ valid: true, actions: 1, secrets: 0 });
});
it.each([
[
{ secret_id: "token", environment_variable: "FIRST" },
@@ -338,11 +296,7 @@ cliTest.for(["launch", "connect"] as const)(
browser:
mode === "launch"
? { mode, executable_path: "/nonexistent/rea-validation-browser" }
: {
mode,
cdp_endpoint: "http://127.0.0.1:1",
target_id: "not-attached",
},
: disconnectedBrowser,
secrets: [
{
secret_id: "session_token",
@@ -410,6 +364,7 @@ cliTest(
issues: [
expect.objectContaining({
path: ["actions"],
reason: "invalid_value",
message: "Action step IDs must be unique",
}),
],
@@ -17,7 +17,7 @@ afterAll(() => schemaValidation.clear());
const scenarios = [
{
name: "undeclared secret references",
fullSchemaValid: true,
canonicalRefinement: true,
issue: {
path: ["secrets"],
reason: "invalid_value",
@@ -34,7 +34,7 @@ const scenarios = [
},
{
name: "duplicate action step IDs",
fullSchemaValid: true,
canonicalRefinement: true,
issue: {
path: ["actions"],
reason: "invalid_value",
@@ -47,7 +47,7 @@ const scenarios = [
},
{
name: "overflowing wait duration",
fullSchemaValid: false,
canonicalRefinement: false,
issue: {
path: ["actions", 0, "duration_ms"],
reason: "out_of_range",
@@ -63,7 +63,7 @@ const scenarios = [
},
{
name: "overflowing action deadline",
fullSchemaValid: false,
canonicalRefinement: false,
issue: {
path: ["actions", 0, "timeout_ms"],
reason: "out_of_range",
@@ -81,9 +81,9 @@ const scenarios = [
},
] as const;
cliTest.for(["compact"] as const)(
"projects browser scenario input diagnostics over stdio MCP (%s)",
async (profile, { cli }) => {
cliTest(
"projects browser scenario input diagnostics over stdio MCP (compact)",
async ({ cli }) => {
const home = await createTestTempDirectory(
"rea-browser-scenario-mcp-home-",
);
@@ -102,7 +102,7 @@ cliTest.for(["compact"] as const)(
PATH: process.env.PATH ?? "",
HOME: home,
REA_LOG_LEVEL: "silent",
REA_MCP_INPUT_SCHEMA_PROFILE: profile,
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
},
stderr: "pipe",
});
@@ -132,7 +132,7 @@ cliTest.for(["compact"] as const)(
expect(
validate(input),
`${scenario.name}: ${JSON.stringify(validate.errors)}`,
).toBe(scenario.fullSchemaValid || profile === "compact");
).toBe(true);
const cliResult = await cli.run({
arguments: [
@@ -171,7 +171,8 @@ cliTest.for(["compact"] as const)(
});
expect(mcpResult.structuredContent).toBeUndefined();
const mcpError = parseMcpToolError(mcpResult).error;
if (scenario.fullSchemaValid) expect(mcpError).toEqual(cliDiagnostic);
if (scenario.canonicalRefinement)
expect(mcpError).toEqual(cliDiagnostic);
else {
// The SDK validates the canonical schema before invoking the handler.
expect(mcpError).toMatchObject({