From 4104f88b627c41e1244c5273d7616801d948db76 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:36:58 +0800 Subject: [PATCH] fix(native-ui): advertise capture source requirement Refs #7570 Co-Authored-By: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- src/cli/utilityCommands.ts | 10 +-- src/contracts/native/nativeToolContracts.ts | 4 +- src/domain/native/nativeUiObservation.ts | 14 +++- src/native/NativeUiObservation.test.ts | 7 +- .../analysisSnapshotDiagnostics.test.ts | 2 +- .../cli/nativeUiScenarioInputCli.test.ts | 70 +++++++++++++++++++ tests/boundary/mcp/nativeUiFactsMcp.test.ts | 36 ++++++++++ 7 files changed, 126 insertions(+), 17 deletions(-) diff --git a/src/cli/utilityCommands.ts b/src/cli/utilityCommands.ts index c431b3cf4e..244ce3c768 100644 --- a/src/cli/utilityCommands.ts +++ b/src/cli/utilityCommands.ts @@ -63,14 +63,8 @@ const nativeUiScopeOptions = { "Exact selected application window ID", ), ), - screenshot: z - .boolean() - .default(true) - .describe("Capture only the selected window"), - accessibility: z - .boolean() - .default(true) - .describe("Read the selected window accessibility tree"), + screenshot: nativeUiScopeFields.screenshot, + accessibility: nativeUiScopeFields.accessibility, maxNodes: cliDecimalField( nativeUiScopeFields.max_nodes.describe( "Maximum accessibility nodes per capture", diff --git a/src/contracts/native/nativeToolContracts.ts b/src/contracts/native/nativeToolContracts.ts index 955a493357..10c082567c 100644 --- a/src/contracts/native/nativeToolContracts.ts +++ b/src/contracts/native/nativeToolContracts.ts @@ -85,12 +85,12 @@ export const swiftSymbolsSchema = z.array( export const NATIVE_TOOL_CONTRACTS = [ native( "observe_native_ui", - "Observe one already-running native app window by PID/window ID, bound to the active Mach-O target. Returns a bounded accessibility tree and window screenshot; permission denial does not broaden capture or launch the target. native_ui_deadline_ms sets this call's deadline independently of output and wait-step limits. Use capture_native_ui_scenario to interact.", + "Observe one already-running native app window by PID/window ID, bound to the active Mach-O target. Returns a bounded accessibility tree and window screenshot; screenshot or accessibility must be enabled. Permission denial does not broaden capture or launch the target. native_ui_deadline_ms sets this call's deadline independently of output and wait-step limits. Use capture_native_ui_scenario to interact.", nativeUiObservationInputSchema, ), native( "capture_native_ui_scenario", - "Interact with one exact native app window through selected-element press, scroll, AXValue text entry and bounded waits. Returns ordered before/after captures and per-step failures. Actions may change app data, network activity and persistent state; REA leaves that state in place. native_ui_deadline_ms controls the scenario deadline, independently of output allowance and wait-step caps. Use observe_native_ui for observation only.", + "Interact with one exact native app window through selected-element press, scroll, AXValue text entry and bounded waits. Returns ordered before/after captures and per-step failures; screenshot or accessibility must be enabled. Actions may change app data, network activity and persistent state; REA leaves that state in place. native_ui_deadline_ms controls the scenario deadline, independently of output allowance and wait-step caps. Use observe_native_ui for observation only.", nativeUiScenarioInputSchema, ), native( diff --git a/src/domain/native/nativeUiObservation.ts b/src/domain/native/nativeUiObservation.ts index f209818dde..78c22e2522 100644 --- a/src/domain/native/nativeUiObservation.ts +++ b/src/domain/native/nativeUiObservation.ts @@ -13,8 +13,18 @@ import { readPngHeaderDimensions } from "../pngHeader.js"; const scope = { pid: z.int().positive(), window_id: z.int().positive(), - screenshot: z.boolean().default(true), - accessibility: z.boolean().default(true), + screenshot: z + .boolean() + .default(true) + .describe( + "Capture the selected window; screenshot or accessibility must be enabled.", + ), + accessibility: z + .boolean() + .default(true) + .describe( + "Read the selected window accessibility tree; screenshot or accessibility must be enabled.", + ), max_nodes: z .int() .positive() diff --git a/src/native/NativeUiObservation.test.ts b/src/native/NativeUiObservation.test.ts index d6f61ef6ed..90c0caf9b6 100644 --- a/src/native/NativeUiObservation.test.ts +++ b/src/native/NativeUiObservation.test.ts @@ -229,10 +229,9 @@ it.each(["observe_native_ui", "capture_native_ui_scenario"] as const)( ], }, }); - expect(projected.message).toContain("both cannot be disabled"); - expect(projected.remediation.action).toContain( - "Set screenshot or accessibility to true", - ); + expect(projected.message).toMatch(/screenshot.*accessibility/iu); + expect(projected.remediation.action).toMatch(/screenshot.*accessibility/iu); + expect(projected.remediation.action).toMatch(/true/iu); }, ); diff --git a/tests/acceptance/analysisSnapshotDiagnostics.test.ts b/tests/acceptance/analysisSnapshotDiagnostics.test.ts index b81bc38ca7..9a4b7d3db4 100644 --- a/tests/acceptance/analysisSnapshotDiagnostics.test.ts +++ b/tests/acceptance/analysisSnapshotDiagnostics.test.ts @@ -29,7 +29,7 @@ cliTest( ] as const) { const encoded = JSON.stringify(document); await writeFile(snapshot, encoded); - const output = await cli.sharedCliRun!({ + const output = await cli.run({ arguments: [ "analyze", target, diff --git a/tests/boundary/cli/nativeUiScenarioInputCli.test.ts b/tests/boundary/cli/nativeUiScenarioInputCli.test.ts index 12ce7a2e13..c76c17851a 100644 --- a/tests/boundary/cli/nativeUiScenarioInputCli.test.ts +++ b/tests/boundary/cli/nativeUiScenarioInputCli.test.ts @@ -21,6 +21,76 @@ const runCapture = async (cli: TestCli, steps: string | undefined) => { }; describe("native UI scenario CLI input", () => { + cliTest( + "advertises the capture source requirement for both commands", + async ({ cli }) => { + for (const command of [ + "observe-native-ui", + "capture-native-ui-scenario", + ]) { + const result = await cli.run({ + arguments: [command, "--schema", "--json"], + }); + expect(result.exitCode).toBe(0); + expect(result.json).toMatchObject({ + options: { + properties: { + screenshot: { + description: expect.stringMatching( + /screenshot or accessibility must be enabled/iu, + ), + }, + accessibility: { + description: expect.stringMatching( + /screenshot or accessibility must be enabled/iu, + ), + }, + }, + }, + }); + } + }, + ); + cliTest.skipIf(process.platform !== "darwin")( + "explains disabled capture sources for observation and scenarios", + async ({ cli }) => { + for (const command of [ + "observe-native-ui", + "capture-native-ui-scenario", + ]) { + const args = [ + command, + "/usr/bin/true", + "--pid", + "1", + "--window-id", + "1", + "--screenshot=false", + "--accessibility=false", + "--json", + ]; + if (command === "capture-native-ui-scenario") + args.push("--steps", '[{"kind":"wait","milliseconds":0}]'); + const result = await cli.run({ + arguments: args, + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + expect.objectContaining({ path: ["screenshot"] }), + expect.objectContaining({ path: ["accessibility"] }), + ], + }, + remediation: { + action: expect.stringMatching(/screenshot.*accessibility/iu), + }, + }); + } + }, + ); cliTest( "rejects omitted steps at CLI argument admission", async ({ cli }) => { diff --git a/tests/boundary/mcp/nativeUiFactsMcp.test.ts b/tests/boundary/mcp/nativeUiFactsMcp.test.ts index aee838457b..a5dcf58e69 100644 --- a/tests/boundary/mcp/nativeUiFactsMcp.test.ts +++ b/tests/boundary/mcp/nativeUiFactsMcp.test.ts @@ -94,6 +94,42 @@ const connectNativeUi = async () => { return { client, replies, snapshot }; }; +it("advertises and explains the capture source requirement through both MCP tools", async () => { + const { client } = await connectNativeUi(); + const tools = (await client.listTools()).tools; + for (const name of [ + "observe_native_ui", + "capture_native_ui_scenario", + ] as const) { + const tool = tools.find((entry) => entry.name === name); + expect(tool).toBeDefined(); + expect(tool?.description).toMatch( + /screenshot or accessibility must be enabled/iu, + ); + const response = await client.callTool({ + name, + arguments: { + ...scope, + accessibility: false, + ...(name === "capture_native_ui_scenario" + ? { steps: [{ kind: "wait", milliseconds: 0 }] } + : {}), + }, + }); + expect(parseMcpToolError(response)).toMatchObject({ + error: { + code: "invalid_request", + details: { + issues: [ + expect.objectContaining({ path: ["screenshot"] }), + expect.objectContaining({ path: ["accessibility"] }), + ], + }, + }, + }); + } +}); + it("preserves scalar accessibility values in passive and scenario Evidence", async () => { const { client, replies, snapshot } = await connectNativeUi(); replies.push(snapshot([0.25, true, "selected", null]));