mirror of
https://github.com/morluto/rea.git
synced 2026-11-14 18:52:48 +00:00
fix(native-ui): advertise capture source requirement
Refs #7570 Co-Authored-By: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
This commit is contained in:
1 parent
60a59daaa6
commit
4104f88b62
7 files changed
+126
-17
No files matched your search
@@ -63,14 +63,8 @@ const nativeUiScopeOptions = {
|
||||
"Exact selected application window ID",
|
||||
),
|
||||
),
|
||||
screenshot: z
|
||||
.boolean()
|
||||
.default(true)
|
||||
.describe("Capture only the selected window"),
|
||||
accessibility: z
|
||||
.boolean()
|
||||
.default(true)
|
||||
.describe("Read the selected window accessibility tree"),
|
||||
screenshot: nativeUiScopeFields.screenshot,
|
||||
accessibility: nativeUiScopeFields.accessibility,
|
||||
maxNodes: cliDecimalField(
|
||||
nativeUiScopeFields.max_nodes.describe(
|
||||
"Maximum accessibility nodes per capture",
|
||||
|
||||
@@ -85,12 +85,12 @@ export const swiftSymbolsSchema = z.array(
|
||||
export const NATIVE_TOOL_CONTRACTS = [
|
||||
native(
|
||||
"observe_native_ui",
|
||||
"Observe one already-running native app window by PID/window ID, bound to the active Mach-O target. Returns a bounded accessibility tree and window screenshot; permission denial does not broaden capture or launch the target. native_ui_deadline_ms sets this call's deadline independently of output and wait-step limits. Use capture_native_ui_scenario to interact.",
|
||||
"Observe one already-running native app window by PID/window ID, bound to the active Mach-O target. Returns a bounded accessibility tree and window screenshot; screenshot or accessibility must be enabled. Permission denial does not broaden capture or launch the target. native_ui_deadline_ms sets this call's deadline independently of output and wait-step limits. Use capture_native_ui_scenario to interact.",
|
||||
nativeUiObservationInputSchema,
|
||||
),
|
||||
native(
|
||||
"capture_native_ui_scenario",
|
||||
"Interact with one exact native app window through selected-element press, scroll, AXValue text entry and bounded waits. Returns ordered before/after captures and per-step failures. Actions may change app data, network activity and persistent state; REA leaves that state in place. native_ui_deadline_ms controls the scenario deadline, independently of output allowance and wait-step caps. Use observe_native_ui for observation only.",
|
||||
"Interact with one exact native app window through selected-element press, scroll, AXValue text entry and bounded waits. Returns ordered before/after captures and per-step failures; screenshot or accessibility must be enabled. Actions may change app data, network activity and persistent state; REA leaves that state in place. native_ui_deadline_ms controls the scenario deadline, independently of output allowance and wait-step caps. Use observe_native_ui for observation only.",
|
||||
nativeUiScenarioInputSchema,
|
||||
),
|
||||
native(
|
||||
|
||||
@@ -13,8 +13,18 @@ import { readPngHeaderDimensions } from "../pngHeader.js";
|
||||
const scope = {
|
||||
pid: z.int().positive(),
|
||||
window_id: z.int().positive(),
|
||||
screenshot: z.boolean().default(true),
|
||||
accessibility: z.boolean().default(true),
|
||||
screenshot: z
|
||||
.boolean()
|
||||
.default(true)
|
||||
.describe(
|
||||
"Capture the selected window; screenshot or accessibility must be enabled.",
|
||||
),
|
||||
accessibility: z
|
||||
.boolean()
|
||||
.default(true)
|
||||
.describe(
|
||||
"Read the selected window accessibility tree; screenshot or accessibility must be enabled.",
|
||||
),
|
||||
max_nodes: z
|
||||
.int()
|
||||
.positive()
|
||||
|
||||
@@ -229,10 +229,9 @@ it.each(["observe_native_ui", "capture_native_ui_scenario"] as const)(
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(projected.message).toContain("both cannot be disabled");
|
||||
expect(projected.remediation.action).toContain(
|
||||
"Set screenshot or accessibility to true",
|
||||
);
|
||||
expect(projected.message).toMatch(/screenshot.*accessibility/iu);
|
||||
expect(projected.remediation.action).toMatch(/screenshot.*accessibility/iu);
|
||||
expect(projected.remediation.action).toMatch(/true/iu);
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ cliTest(
|
||||
] as const) {
|
||||
const encoded = JSON.stringify(document);
|
||||
await writeFile(snapshot, encoded);
|
||||
const output = await cli.sharedCliRun!({
|
||||
const output = await cli.run({
|
||||
arguments: [
|
||||
"analyze",
|
||||
target,
|
||||
|
||||
@@ -21,6 +21,76 @@ const runCapture = async (cli: TestCli, steps: string | undefined) => {
|
||||
};
|
||||
|
||||
describe("native UI scenario CLI input", () => {
|
||||
cliTest(
|
||||
"advertises the capture source requirement for both commands",
|
||||
async ({ cli }) => {
|
||||
for (const command of [
|
||||
"observe-native-ui",
|
||||
"capture-native-ui-scenario",
|
||||
]) {
|
||||
const result = await cli.run({
|
||||
arguments: [command, "--schema", "--json"],
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.json).toMatchObject({
|
||||
options: {
|
||||
properties: {
|
||||
screenshot: {
|
||||
description: expect.stringMatching(
|
||||
/screenshot or accessibility must be enabled/iu,
|
||||
),
|
||||
},
|
||||
accessibility: {
|
||||
description: expect.stringMatching(
|
||||
/screenshot or accessibility must be enabled/iu,
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
cliTest.skipIf(process.platform !== "darwin")(
|
||||
"explains disabled capture sources for observation and scenarios",
|
||||
async ({ cli }) => {
|
||||
for (const command of [
|
||||
"observe-native-ui",
|
||||
"capture-native-ui-scenario",
|
||||
]) {
|
||||
const args = [
|
||||
command,
|
||||
"/usr/bin/true",
|
||||
"--pid",
|
||||
"1",
|
||||
"--window-id",
|
||||
"1",
|
||||
"--screenshot=false",
|
||||
"--accessibility=false",
|
||||
"--json",
|
||||
];
|
||||
if (command === "capture-native-ui-scenario")
|
||||
args.push("--steps", '[{"kind":"wait","milliseconds":0}]');
|
||||
const result = await cli.run({
|
||||
arguments: args,
|
||||
environment: { REA_LOG_LEVEL: "silent" },
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.json).toMatchObject({
|
||||
code: "invalid_request",
|
||||
details: {
|
||||
issues: [
|
||||
expect.objectContaining({ path: ["screenshot"] }),
|
||||
expect.objectContaining({ path: ["accessibility"] }),
|
||||
],
|
||||
},
|
||||
remediation: {
|
||||
action: expect.stringMatching(/screenshot.*accessibility/iu),
|
||||
},
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
cliTest(
|
||||
"rejects omitted steps at CLI argument admission",
|
||||
async ({ cli }) => {
|
||||
|
||||
@@ -94,6 +94,42 @@ const connectNativeUi = async () => {
|
||||
return { client, replies, snapshot };
|
||||
};
|
||||
|
||||
it("advertises and explains the capture source requirement through both MCP tools", async () => {
|
||||
const { client } = await connectNativeUi();
|
||||
const tools = (await client.listTools()).tools;
|
||||
for (const name of [
|
||||
"observe_native_ui",
|
||||
"capture_native_ui_scenario",
|
||||
] as const) {
|
||||
const tool = tools.find((entry) => entry.name === name);
|
||||
expect(tool).toBeDefined();
|
||||
expect(tool?.description).toMatch(
|
||||
/screenshot or accessibility must be enabled/iu,
|
||||
);
|
||||
const response = await client.callTool({
|
||||
name,
|
||||
arguments: {
|
||||
...scope,
|
||||
accessibility: false,
|
||||
...(name === "capture_native_ui_scenario"
|
||||
? { steps: [{ kind: "wait", milliseconds: 0 }] }
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
expect(parseMcpToolError(response)).toMatchObject({
|
||||
error: {
|
||||
code: "invalid_request",
|
||||
details: {
|
||||
issues: [
|
||||
expect.objectContaining({ path: ["screenshot"] }),
|
||||
expect.objectContaining({ path: ["accessibility"] }),
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves scalar accessibility values in passive and scenario Evidence", async () => {
|
||||
const { client, replies, snapshot } = await connectNativeUi();
|
||||
replies.push(snapshot([0.25, true, "selected", null]));
|
||||
|
||||
Reference in new issue
Block a user