fix(native-ui): advertise capture source requirement

Refs #7570

Co-Authored-By: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
This commit is contained in:
morlutoandgensokyo-dev authored and Ysopwnable committed 2026-11-14 14:06:23 +01:00
1 parent 60a59daaa6
commit 4104f88b62
7 files changed
+126 -17

No files matched your search

+2 -8
View File
@@ -63,14 +63,8 @@ const nativeUiScopeOptions = {
"Exact selected application window ID",
),
),
screenshot: z
.boolean()
.default(true)
.describe("Capture only the selected window"),
accessibility: z
.boolean()
.default(true)
.describe("Read the selected window accessibility tree"),
screenshot: nativeUiScopeFields.screenshot,
accessibility: nativeUiScopeFields.accessibility,
maxNodes: cliDecimalField(
nativeUiScopeFields.max_nodes.describe(
"Maximum accessibility nodes per capture",
+2 -2
View File
@@ -85,12 +85,12 @@ export const swiftSymbolsSchema = z.array(
export const NATIVE_TOOL_CONTRACTS = [
native(
"observe_native_ui",
"Observe one already-running native app window by PID/window ID, bound to the active Mach-O target. Returns a bounded accessibility tree and window screenshot; permission denial does not broaden capture or launch the target. native_ui_deadline_ms sets this call's deadline independently of output and wait-step limits. Use capture_native_ui_scenario to interact.",
"Observe one already-running native app window by PID/window ID, bound to the active Mach-O target. Returns a bounded accessibility tree and window screenshot; screenshot or accessibility must be enabled. Permission denial does not broaden capture or launch the target. native_ui_deadline_ms sets this call's deadline independently of output and wait-step limits. Use capture_native_ui_scenario to interact.",
nativeUiObservationInputSchema,
),
native(
"capture_native_ui_scenario",
"Interact with one exact native app window through selected-element press, scroll, AXValue text entry and bounded waits. Returns ordered before/after captures and per-step failures. Actions may change app data, network activity and persistent state; REA leaves that state in place. native_ui_deadline_ms controls the scenario deadline, independently of output allowance and wait-step caps. Use observe_native_ui for observation only.",
"Interact with one exact native app window through selected-element press, scroll, AXValue text entry and bounded waits. Returns ordered before/after captures and per-step failures; screenshot or accessibility must be enabled. Actions may change app data, network activity and persistent state; REA leaves that state in place. native_ui_deadline_ms controls the scenario deadline, independently of output allowance and wait-step caps. Use observe_native_ui for observation only.",
nativeUiScenarioInputSchema,
),
native(
+12 -2
View File
@@ -13,8 +13,18 @@ import { readPngHeaderDimensions } from "../pngHeader.js";
const scope = {
pid: z.int().positive(),
window_id: z.int().positive(),
screenshot: z.boolean().default(true),
accessibility: z.boolean().default(true),
screenshot: z
.boolean()
.default(true)
.describe(
"Capture the selected window; screenshot or accessibility must be enabled.",
),
accessibility: z
.boolean()
.default(true)
.describe(
"Read the selected window accessibility tree; screenshot or accessibility must be enabled.",
),
max_nodes: z
.int()
.positive()
+3 -4
View File
@@ -229,10 +229,9 @@ it.each(["observe_native_ui", "capture_native_ui_scenario"] as const)(
],
},
});
expect(projected.message).toContain("both cannot be disabled");
expect(projected.remediation.action).toContain(
"Set screenshot or accessibility to true",
);
expect(projected.message).toMatch(/screenshot.*accessibility/iu);
expect(projected.remediation.action).toMatch(/screenshot.*accessibility/iu);
expect(projected.remediation.action).toMatch(/true/iu);
},
);
@@ -29,7 +29,7 @@ cliTest(
] as const) {
const encoded = JSON.stringify(document);
await writeFile(snapshot, encoded);
const output = await cli.sharedCliRun!({
const output = await cli.run({
arguments: [
"analyze",
target,
@@ -21,6 +21,76 @@ const runCapture = async (cli: TestCli, steps: string | undefined) => {
};
describe("native UI scenario CLI input", () => {
cliTest(
"advertises the capture source requirement for both commands",
async ({ cli }) => {
for (const command of [
"observe-native-ui",
"capture-native-ui-scenario",
]) {
const result = await cli.run({
arguments: [command, "--schema", "--json"],
});
expect(result.exitCode).toBe(0);
expect(result.json).toMatchObject({
options: {
properties: {
screenshot: {
description: expect.stringMatching(
/screenshot or accessibility must be enabled/iu,
),
},
accessibility: {
description: expect.stringMatching(
/screenshot or accessibility must be enabled/iu,
),
},
},
},
});
}
},
);
cliTest.skipIf(process.platform !== "darwin")(
"explains disabled capture sources for observation and scenarios",
async ({ cli }) => {
for (const command of [
"observe-native-ui",
"capture-native-ui-scenario",
]) {
const args = [
command,
"/usr/bin/true",
"--pid",
"1",
"--window-id",
"1",
"--screenshot=false",
"--accessibility=false",
"--json",
];
if (command === "capture-native-ui-scenario")
args.push("--steps", '[{"kind":"wait","milliseconds":0}]');
const result = await cli.run({
arguments: args,
environment: { REA_LOG_LEVEL: "silent" },
});
expect(result.exitCode).toBe(1);
expect(result.json).toMatchObject({
code: "invalid_request",
details: {
issues: [
expect.objectContaining({ path: ["screenshot"] }),
expect.objectContaining({ path: ["accessibility"] }),
],
},
remediation: {
action: expect.stringMatching(/screenshot.*accessibility/iu),
},
});
}
},
);
cliTest(
"rejects omitted steps at CLI argument admission",
async ({ cli }) => {
@@ -94,6 +94,42 @@ const connectNativeUi = async () => {
return { client, replies, snapshot };
};
it("advertises and explains the capture source requirement through both MCP tools", async () => {
const { client } = await connectNativeUi();
const tools = (await client.listTools()).tools;
for (const name of [
"observe_native_ui",
"capture_native_ui_scenario",
] as const) {
const tool = tools.find((entry) => entry.name === name);
expect(tool).toBeDefined();
expect(tool?.description).toMatch(
/screenshot or accessibility must be enabled/iu,
);
const response = await client.callTool({
name,
arguments: {
...scope,
accessibility: false,
...(name === "capture_native_ui_scenario"
? { steps: [{ kind: "wait", milliseconds: 0 }] }
: {}),
},
});
expect(parseMcpToolError(response)).toMatchObject({
error: {
code: "invalid_request",
details: {
issues: [
expect.objectContaining({ path: ["screenshot"] }),
expect.objectContaining({ path: ["accessibility"] }),
],
},
},
});
}
});
it("preserves scalar accessibility values in passive and scenario Evidence", async () => {
const { client, replies, snapshot } = await connectNativeUi();
replies.push(snapshot([0.25, true, "selected", null]));