mirror of
https://github.com/morluto/rea.git
synced 2026-11-22 14:35:45 +00:00
test(web): consolidate scenario input diagnostics (#13159)
This commit is contained in:
1 parent
52437cd431
commit
0fa220c735
3 files changed
+47
-116
No files matched your search
@@ -2,7 +2,6 @@ import { createHash } from "node:crypto";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { BrowserScenarioSecrets } from "./BrowserScenarioSecrets.js";
|
||||
import { resolveScenarioSecrets } from "../../tests/fixtures/browserScenarioSecrets.js";
|
||||
import { browserScenarioSchema } from "../domain/browserScenario.js";
|
||||
import { DEFAULT_BROWSER_SCENARIO_CAPTURE_LIMITS } from "../domain/browserScenarioCaptureLimits.js";
|
||||
@@ -125,30 +124,6 @@ describe("browser scenario secret redaction", () => {
|
||||
value_sha256: null,
|
||||
});
|
||||
});
|
||||
it("does not resolve inherited environment properties as secret values", () => {
|
||||
const scenario = browserScenarioSchema.parse(baseScenario());
|
||||
const environment = Object.assign(
|
||||
Object.create({ REA_TEST_SESSION: "session" }) as Record<string, string>,
|
||||
{ REA_TEST_PASSWORD: "password", REA_TEST_COOKIE: "cookie" },
|
||||
);
|
||||
expect(BrowserScenarioSecrets.resolve(scenario, environment)).toMatchObject(
|
||||
{
|
||||
ok: false,
|
||||
error: {
|
||||
_tag: "AnalysisInputError",
|
||||
issues: [
|
||||
{
|
||||
path: ["secrets", 0, "environment_variable"],
|
||||
expected: {
|
||||
environment_variable: "REA_TEST_SESSION",
|
||||
secret_id: "session",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("storage value fingerprint encoding", () => {
|
||||
|
||||
@@ -53,15 +53,6 @@ const launchScenario = {
|
||||
actions: [{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 }],
|
||||
} as const;
|
||||
|
||||
it("accepts a valid scenario without launching or attaching to a browser", async () => {
|
||||
const output = await serveJson(
|
||||
registerValidatingCli(),
|
||||
"validate-browser-scenario",
|
||||
launchScenario,
|
||||
);
|
||||
expect(output).toEqual({ valid: true, actions: 1, secrets: 0 });
|
||||
});
|
||||
|
||||
it.each(["validate-browser-scenario", "capture-browser-scenario"])(
|
||||
"%s rejects a literal Basic username containing a colon before creating a provider",
|
||||
async (command) => {
|
||||
@@ -104,53 +95,40 @@ it.each(["validate-browser-scenario", "capture-browser-scenario"])(
|
||||
},
|
||||
);
|
||||
|
||||
it("counts declared secrets without resolving their environment values", async () => {
|
||||
const cli = registerValidatingCli();
|
||||
const output = await serveJson(cli, "validate-browser-scenario", {
|
||||
...launchScenario,
|
||||
actions: [
|
||||
{
|
||||
step_id: "sign-in",
|
||||
action: "fill",
|
||||
locator: { kind: "css", selector: "#token" },
|
||||
value: { source: "secret", secret_id: "session_token" },
|
||||
},
|
||||
],
|
||||
secrets: [
|
||||
{ secret_id: "session_token", environment_variable: "SESSION_TOKEN" },
|
||||
{ secret_id: "api_key", environment_variable: "API_KEY" },
|
||||
],
|
||||
});
|
||||
expect(output).toEqual({ valid: true, actions: 1, secrets: 2 });
|
||||
});
|
||||
const secretAction = {
|
||||
step_id: "sign-in",
|
||||
action: "fill",
|
||||
locator: { kind: "css", selector: "#token" },
|
||||
value: { source: "secret", secret_id: "session_token" },
|
||||
};
|
||||
const disconnectedBrowser = {
|
||||
mode: "connect",
|
||||
cdp_endpoint: "http://127.0.0.1:1",
|
||||
target_id: "unavailable-target",
|
||||
};
|
||||
|
||||
it("rejects a duplicate action step identity with the offending field path", async () => {
|
||||
const output = await serveJson(
|
||||
registerValidatingCli(),
|
||||
"validate-browser-scenario",
|
||||
{
|
||||
...launchScenario,
|
||||
actions: [
|
||||
{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 },
|
||||
{ step_id: "settle", action: "wait_for_timeout", duration_ms: 2 },
|
||||
],
|
||||
},
|
||||
);
|
||||
expect(output).toMatchObject({
|
||||
error: "Browser scenario validation failed",
|
||||
code: "invalid_request",
|
||||
category: "invalid_input",
|
||||
details: {
|
||||
issues: [
|
||||
expect.objectContaining({
|
||||
path: ["actions"],
|
||||
reason: "invalid_value",
|
||||
message: "Action step IDs must be unique",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
it.each([
|
||||
["launch", launchScenario.browser],
|
||||
["connect", disconnectedBrowser],
|
||||
])(
|
||||
"validates %s and counts secrets without touching runtime resources",
|
||||
async (_mode, browser) => {
|
||||
const output = await serveJson(
|
||||
registerValidatingCli(),
|
||||
"validate-browser-scenario",
|
||||
{
|
||||
...launchScenario,
|
||||
browser,
|
||||
actions: [secretAction],
|
||||
secrets: [
|
||||
{ secret_id: "session_token", environment_variable: "SESSION_TOKEN" },
|
||||
{ secret_id: "api_key", environment_variable: "API_KEY" },
|
||||
],
|
||||
},
|
||||
);
|
||||
expect(output).toEqual({ valid: true, actions: 1, secrets: 2 });
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects an action that reuses the reserved initial step identity", async () => {
|
||||
const output = await serveJson(
|
||||
@@ -186,14 +164,7 @@ it("rejects an undeclared secret reference without exposing any secret value", a
|
||||
const cli = registerValidatingCli();
|
||||
const output = await serveJson(cli, "validate-browser-scenario", {
|
||||
...launchScenario,
|
||||
actions: [
|
||||
{
|
||||
step_id: "sign-in",
|
||||
action: "fill",
|
||||
locator: { kind: "css", selector: "#token" },
|
||||
value: { source: "secret", secret_id: "session_token" },
|
||||
},
|
||||
],
|
||||
actions: [secretAction],
|
||||
});
|
||||
const serialized = JSON.stringify(output);
|
||||
expect(output).toMatchObject({
|
||||
@@ -240,19 +211,6 @@ it("rejects an unknown field instead of ignoring it", async () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts a disconnected CDP target without constructing a provider", async () => {
|
||||
expect(
|
||||
await serveJson(registerValidatingCli(), "validate-browser-scenario", {
|
||||
...launchScenario,
|
||||
browser: {
|
||||
mode: "connect",
|
||||
cdp_endpoint: "http://127.0.0.1:1",
|
||||
target_id: "unavailable-target",
|
||||
},
|
||||
}),
|
||||
).toEqual({ valid: true, actions: 1, secrets: 0 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
{ secret_id: "token", environment_variable: "FIRST" },
|
||||
@@ -338,11 +296,7 @@ cliTest.for(["launch", "connect"] as const)(
|
||||
browser:
|
||||
mode === "launch"
|
||||
? { mode, executable_path: "/nonexistent/rea-validation-browser" }
|
||||
: {
|
||||
mode,
|
||||
cdp_endpoint: "http://127.0.0.1:1",
|
||||
target_id: "not-attached",
|
||||
},
|
||||
: disconnectedBrowser,
|
||||
secrets: [
|
||||
{
|
||||
secret_id: "session_token",
|
||||
@@ -410,6 +364,7 @@ cliTest(
|
||||
issues: [
|
||||
expect.objectContaining({
|
||||
path: ["actions"],
|
||||
reason: "invalid_value",
|
||||
message: "Action step IDs must be unique",
|
||||
}),
|
||||
],
|
||||
|
||||
@@ -17,7 +17,7 @@ afterAll(() => schemaValidation.clear());
|
||||
const scenarios = [
|
||||
{
|
||||
name: "undeclared secret references",
|
||||
fullSchemaValid: true,
|
||||
canonicalRefinement: true,
|
||||
issue: {
|
||||
path: ["secrets"],
|
||||
reason: "invalid_value",
|
||||
@@ -34,7 +34,7 @@ const scenarios = [
|
||||
},
|
||||
{
|
||||
name: "duplicate action step IDs",
|
||||
fullSchemaValid: true,
|
||||
canonicalRefinement: true,
|
||||
issue: {
|
||||
path: ["actions"],
|
||||
reason: "invalid_value",
|
||||
@@ -47,7 +47,7 @@ const scenarios = [
|
||||
},
|
||||
{
|
||||
name: "overflowing wait duration",
|
||||
fullSchemaValid: false,
|
||||
canonicalRefinement: false,
|
||||
issue: {
|
||||
path: ["actions", 0, "duration_ms"],
|
||||
reason: "out_of_range",
|
||||
@@ -63,7 +63,7 @@ const scenarios = [
|
||||
},
|
||||
{
|
||||
name: "overflowing action deadline",
|
||||
fullSchemaValid: false,
|
||||
canonicalRefinement: false,
|
||||
issue: {
|
||||
path: ["actions", 0, "timeout_ms"],
|
||||
reason: "out_of_range",
|
||||
@@ -81,9 +81,9 @@ const scenarios = [
|
||||
},
|
||||
] as const;
|
||||
|
||||
cliTest.for(["compact"] as const)(
|
||||
"projects browser scenario input diagnostics over stdio MCP (%s)",
|
||||
async (profile, { cli }) => {
|
||||
cliTest(
|
||||
"projects browser scenario input diagnostics over stdio MCP (compact)",
|
||||
async ({ cli }) => {
|
||||
const home = await createTestTempDirectory(
|
||||
"rea-browser-scenario-mcp-home-",
|
||||
);
|
||||
@@ -102,7 +102,7 @@ cliTest.for(["compact"] as const)(
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: home,
|
||||
REA_LOG_LEVEL: "silent",
|
||||
REA_MCP_INPUT_SCHEMA_PROFILE: profile,
|
||||
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
@@ -132,7 +132,7 @@ cliTest.for(["compact"] as const)(
|
||||
expect(
|
||||
validate(input),
|
||||
`${scenario.name}: ${JSON.stringify(validate.errors)}`,
|
||||
).toBe(scenario.fullSchemaValid || profile === "compact");
|
||||
).toBe(true);
|
||||
|
||||
const cliResult = await cli.run({
|
||||
arguments: [
|
||||
@@ -171,7 +171,8 @@ cliTest.for(["compact"] as const)(
|
||||
});
|
||||
expect(mcpResult.structuredContent).toBeUndefined();
|
||||
const mcpError = parseMcpToolError(mcpResult).error;
|
||||
if (scenario.fullSchemaValid) expect(mcpError).toEqual(cliDiagnostic);
|
||||
if (scenario.canonicalRefinement)
|
||||
expect(mcpError).toEqual(cliDiagnostic);
|
||||
else {
|
||||
// The SDK validates the canonical schema before invoking the handler.
|
||||
expect(mcpError).toMatchObject({
|
||||
|
||||
Reference in new issue
Block a user