diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57b3c6d32a..6880245b92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -807,6 +807,8 @@ jobs: tests/boundary/mcp/nativeDemangleMcp.test.ts tests/boundary/mcp/nativeMachoSummaryMcp.test.ts tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts + tests/boundary/providers/native/nativeUiOffscreenWindow.test.ts + tests/boundary/process/nativeUiVerifierLane.test.ts tests/boundary/cli/nativeSignatureCli.test.ts tests/boundary/cli/nativeMachoArm64_32Cli.test.ts tests/boundary/cli/nativeMachoEncryptionCli.test.ts diff --git a/bridge/native/ReaNativeUI.swift b/bridge/native/ReaNativeUI.swift index b123a57f24..2c7bd73992 100644 --- a/bridge/native/ReaNativeUI.swift +++ b/bridge/native/ReaNativeUI.swift @@ -65,7 +65,10 @@ func observe(_ request: Request) async throws -> [String: Any] { guard SHA256.hash(data: executableBytes).map({ String(format: "%02x", $0) }).joined() == request.sha256 else { try fail("target-mismatch", "Running executable bytes differ from the active target digest") } - guard let windows = CGWindowListCopyWindowInfo([.optionIncludingWindow], request.window_id) as? [[String: Any]], + // .optionIncludingWindow omits a selected window while it is off screen, which + // is not absence: select from every window, and let capture report what it can. + guard let windows = (CGWindowListCopyWindowInfo([.optionAll], kCGNullWindowID) as? [[String: Any]])? + .filter({ ($0[kCGWindowNumber as String] as? NSNumber)?.uint32Value == request.window_id }), windows.count == 1, let window = windows.first, (window[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == request.pid, (window[kCGWindowLayer as String] as? NSNumber)?.intValue == 0, diff --git a/docs/native-investigation.md b/docs/native-investigation.md index fe02a2b9d4..abd1424ea5 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -1004,7 +1004,9 @@ with a path-specific gap and incomplete capture coverage. Null alone does not establish that an accessibility value was successfully read or absent. An initial PID or window mismatch is `invalid_request` on `pid` or -`window_id` and leaves provider availability intact. An unavailable executable +`window_id` and leaves provider availability intact. A selected window that +exists off screen is not a mismatch: selection succeeds and the permission and +capture checks report their own outcome. An unavailable executable URL or unresolved executable path leaves identity unknown and reports `capability_unavailable`. Identity or window loss after the initial capture remains an observation failure with collected facts. diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 051406c625..7f5e4d3839 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -165,6 +165,9 @@ const PATH_SCOPES = { "tests/boundary/mcp/nativeDemangleMcp.test.ts", "tests/boundary/mcp/nativeMachoSummaryMcp.test.ts", "tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts", + "tests/boundary/providers/native/nativeUiOffscreenWindow.test.ts", + "tests/boundary/process/nativeUiVerifierLane.test.ts", + "tests/conformance/native/ui-offscreen.swift", "tests/boundary/cli/nativeSignatureCli.test.ts", "tests/boundary/cli/nativeMachoArm64_32Cli.test.ts", "tests/boundary/cli/nativeMachoEncryptionCli.test.ts", diff --git a/scripts/verify-native-ui.mjs b/scripts/verify-native-ui.mjs index 91a9f6a169..b7bfb43536 100644 --- a/scripts/verify-native-ui.mjs +++ b/scripts/verify-native-ui.mjs @@ -245,7 +245,13 @@ try { scope, { environment: process.env }, ); - if (mismatch.ok || !mismatch.error.message.includes("target-mismatch")) + // A selection failure carries its helper code in the issue, not in its message. + if ( + mismatch.ok || + !mismatch.error.issues?.some(({ message }) => + message.startsWith("target-mismatch:"), + ) + ) throw new Error("Real helper did not reject changed target bytes"); report = { ok: true, diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 1be899b47f..351223eb05 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -897,6 +897,18 @@ it.each([ "tests/boundary/cli/nativePlistCli.test.ts", { code: true, apple: true, package: false, runtime: false }, ], + [ + "tests/boundary/process/nativeUiVerifierLane.test.ts", + { code: true, apple: true, package: false, runtime: false }, + ], + [ + "tests/boundary/providers/native/nativeUiOffscreenWindow.test.ts", + { code: true, apple: true, package: false, runtime: false }, + ], + [ + "tests/conformance/native/ui-offscreen.swift", + { code: true, apple: true, package: false, runtime: false }, + ], [ "src/artifacts/InesArtifactReader.ts", { code: true, apple: true, package: false, runtime: true }, diff --git a/tests/boundary/process/nativeUiVerifierLane.test.ts b/tests/boundary/process/nativeUiVerifierLane.test.ts new file mode 100644 index 0000000000..5ddf216c58 --- /dev/null +++ b/tests/boundary/process/nativeUiVerifierLane.test.ts @@ -0,0 +1,222 @@ +import { spawn } from "node:child_process"; +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { pathToFileURL } from "node:url"; +import { expect, it, onTestFinished } from "vitest"; +import { z } from "zod"; + +import { systemProcessOwnershipHost } from "../../../src/process/ProcessOwnershipObservation.js"; +import { liveProcesses } from "../../../src/process/ProcessOwnershipProcessTree.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +// Stands in for /usr/bin/open: starts a detached app that inherits its environment. +// Each stand-in records itself and leaves when the test releases it, or expires. +const standInPrelude = ` +import { appendFileSync, existsSync, readFileSync } from "node:fs"; +const config = JSON.parse(readFileSync(new URL("./config.json", import.meta.url), "utf8")); +const record = (kind) => appendFileSync(config.events, JSON.stringify({ kind, pid: process.pid }) + "\\n"); +setInterval(() => { + if (existsSync(config.release)) process.exit(0); +}, 10); +setTimeout(() => process.exit(0), 60000); +`; + +const launcherSource = ` +import { spawn } from "node:child_process"; +import { openSync } from "node:fs"; +${standInPrelude} +record("launcher"); +const args = process.argv.slice(2); +const coordinates = args[args.indexOf("--stdout") + 1]; +spawn(process.execPath, [new URL("./fixture.mjs", import.meta.url).pathname], { + detached: true, + stdio: ["ignore", openSync(coordinates, "a"), "ignore"], +}).unref(); +`; + +const fixtureSource = ` +import { writeSync } from "node:fs"; +${standInPrelude} +record("fixture"); +writeSync(1, JSON.stringify({ pid: process.pid, window_id: 7 }) + "\\n"); +`; + +// Runs the actual verifier and observation mapping. Only the compiler and launcher +// are replaced: the compiled helper is an authored script answering as the real +// helper does, so this covers the lane's assertions and not native capture. +const preloadSource = ` +import { chmodSync, copyFileSync, readFileSync, writeFileSync } from "node:fs"; +import { createRequire, syncBuiltinESMExports } from "node:module"; +import { basename } from "node:path"; +import { promisify } from "node:util"; +const config = JSON.parse(readFileSync(new URL("./config.json", import.meta.url), "utf8")); +const childProcess = createRequire(import.meta.url)("node:child_process"); +const { execFile, spawn } = childProcess; +const script = (path, body) => { + writeFileSync(path, "#!/bin/sh\\n" + body + "\\n"); + chmodSync(path, 0o755); +}; +const compile = (args) => { + const output = args[args.indexOf("-o") + 1]; + if (basename(output) === "fixture") copyFileSync("/usr/bin/true", output); + else if (basename(output) === "child-retrieval-test") + script(output, "printf '%s\\\\n' '{\\"a\\":\\"b\\"}' '{\\"n\\":1.0}'"); + else script(output, config.helper); +}; +const compiled = (file, args) => file === "/usr/bin/xcrun" && args.includes("-o"); +childProcess.execFile = (file, args, ...rest) => { + if (!compiled(file, args)) return execFile(file, args, ...rest); + compile(args); + const done = rest.at(-1); + queueMicrotask(() => done(null, "", "")); + return spawn("/usr/bin/true", [], { stdio: "ignore" }); +}; +childProcess.execFile[promisify.custom] = (file, args, ...rest) => { + if (!compiled(file, args)) return execFile[promisify.custom](file, args, ...rest); + compile(args); + return Promise.resolve({ stdout: "", stderr: "" }); +}; +childProcess.spawn = (file, args, options) => { + if (compiled(file, args)) { + compile(args); + return spawn("/usr/bin/true", [], options); + } + return file === "/usr/bin/open" + ? spawn(process.execPath, [config.launcherEntry, ...args], options) + : spawn(file, args, options); +}; +syncBuiltinESMExports(); +`; + +const denied = `printf '%s' '{"ok":false,"code":"screen-recording-denied","message":"authored permission denial"}'`; +const answersChangedBytes = `case "$1" in *'"sha256":"0000000000'*) printf '%s' '{"ok":false,"code":"target-mismatch","message":"authored digest refusal"}'; exit 0;; esac; ${denied}`; + +const standInSchema = z.object({ + kind: z.enum(["launcher", "fixture"]), + pid: z.number().int().positive(), +}); + +const runVerifier = async (helper: string, flags: readonly string[]) => { + const workspace = await createTestTempDirectory("rea-native-ui-lane-"); + const paths = { + events: join(workspace, "events.jsonl"), + release: join(workspace, "release"), + launcherEntry: join(workspace, "launcher.mjs"), + }; + await writeFile(paths.events, ""); + await writeFile(paths.launcherEntry, launcherSource); + await writeFile(join(workspace, "fixture.mjs"), fixtureSource); + await writeFile(join(workspace, "preload.mjs"), preloadSource); + await writeFile( + join(workspace, "config.json"), + JSON.stringify({ ...paths, helper }), + ); + const child = spawn( + process.execPath, + [ + "--import", + pathToFileURL(join(workspace, "preload.mjs")).href, + resolve("scripts/verify-native-ui.mjs"), + ...flags, + ], + { + env: { ...process.env, TMPDIR: workspace }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + const standIns = async () => + (await readFile(paths.events, "utf8")) + .split("\n") + .filter((line) => line.length > 0) + .map((line) => standInSchema.parse(JSON.parse(line))); + const isLive = async (pid: number) => + liveProcesses(await systemProcessOwnershipHost.listProcesses()).some( + (process) => process.pid === pid, + ); + const untilGone = async (pid: number) => { + const deadline = performance.now() + 15_000; + while ((await isLive(pid)) && performance.now() < deadline) await delay(25); + return !(await isLive(pid)); + }; + // Runs on a timeout or failed assertion too: the verifier cannot release a stand-in + // once killed, and its fixture would otherwise outlive the test until it expired. + onTestFinished(async () => { + child.kill("SIGKILL"); + await writeFile(paths.release, ""); + for (const { pid } of await standIns()) + expect(await untilGone(pid)).toBe(true); + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString("utf8"); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + const code = await new Promise((settle) => { + child.once("close", settle); + }); + const fixture = (await standIns()).find(({ kind }) => kind === "fixture"); + expect(fixture, stderr).toBeDefined(); + return { + code, + stdout, + stderr, + fixtureLive: !(await untilGone(fixture?.pid ?? -1)), + roots: (await readdir(workspace)).filter((name) => + name.startsWith("rea-ui-fixture-"), + ), + }; +}; + +// The verifier refuses other hosts before any fixture is created. +const macosIt = it.skipIf(process.platform !== "darwin"); + +macosIt( + "completes at the permission boundary when the helper refuses changed target bytes", + { timeout: 30_000 }, + async () => { + const result = await runVerifier(answersChangedBytes, [ + "--allow-permission-denial", + ]); + expect(result.code, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + ok: true, + verification_status: "permission-boundary-only", + positive_e2e: false, + mismatch_rejected: true, + }); + expect(result.fixtureLive).toBe(false); + expect(result.roots).toEqual([]); + }, +); + +macosIt( + "fails when the helper does not refuse changed target bytes", + { timeout: 30_000 }, + async () => { + const result = await runVerifier(denied, ["--allow-permission-denial"]); + expect(result.code).toBe(1); + expect(result.stderr).toContain( + "Real helper did not reject changed target bytes", + ); + expect(result.stdout).toBe(""); + expect(result.fixtureLive).toBe(false); + expect(result.roots).toEqual([]); + }, +); + +macosIt( + "keeps a permission denial a failure in the positive lane", + { timeout: 30_000 }, + async () => { + const result = await runVerifier(answersChangedBytes, []); + expect(result.code).toBe(1); + expect(result.stderr).toContain("requires successful capture"); + expect(result.stdout).toBe(""); + expect(result.fixtureLive).toBe(false); + expect(result.roots).toEqual([]); + }, +); diff --git a/tests/boundary/providers/native/nativeUiOffscreenWindow.test.ts b/tests/boundary/providers/native/nativeUiOffscreenWindow.test.ts new file mode 100644 index 0000000000..a2de197863 --- /dev/null +++ b/tests/boundary/providers/native/nativeUiOffscreenWindow.test.ts @@ -0,0 +1,138 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { promisify } from "node:util"; +import { expect, it, onTestFinished } from "vitest"; +import { z } from "zod"; + +import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; +import { AnalysisInputError } from "../../../../src/domain/analysisErrorCore.js"; +import { createNativeUiHelperRuntime } from "../../../../src/native/NativeUiHelperRuntime.js"; +import { observeNativeUi } from "../../../../src/native/NativeUiObservation.js"; +import { cleanupOwnedProcessGroup } from "../../../../src/process/ProcessOwnership.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../../../../src/process/ProviderProcess.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; + +const scopeSchema = z.object({ + pid: z.number().int().positive(), + window_id: z.number().int().positive(), +}); + +// Ownership starts with the launch: the app inherits the launcher's run token, so +// the sweep finds it with or without a ready record. /usr/bin/open's own token is +// withheld, so its retained handle must stop it before the sweep can succeed. +const launch = async (bundle: string, name: string) => { + const coordinates = join(bundle, "..", `${name}.json`); + await writeFile(coordinates, ""); + const owned = await spawnOwnedProviderProcess({ + command: "/usr/bin/open", + arguments: ["-W", "-n", "-g", "--stdout", coordinates, bundle], + runId: randomUUID(), + sweepTokenOwnedProcesses: true, + }); + const launcher = new ProviderProcessSupervisor( + { process: owned.process, ownsProcessLifetime: true }, + { maxDiagnosticBytes: { stdout: 0, stderr: 64 * 1024 } }, + ); + onTestFinished(async () => { + const stopped = await launcher.stop(); + const swept = await cleanupOwnedProcessGroup(owned.ownership); + expect(stopped.status).not.toBe("incomplete"); + expect(swept).toMatchObject({ cleaned: true }); + }); + const deadline = performance.now() + 30_000; + for (;;) { + const [line] = (await readFile(coordinates, "utf8")).split("\n"); + if (line !== undefined && line.length > 0) + return scopeSchema.parse(JSON.parse(line)); + if (performance.now() >= deadline) + throw new Error("The off-screen fixture never reported its window"); + await delay(50); + } +}; + +// The fixture window is real and never ordered in; the helper is compiled and run +// by the production runtime. Capture is off for the raw helper reads, so these +// facts need no Accessibility or Screen Recording permission. +it.skipIf(process.platform !== "darwin")( + "selects a window that exists off screen and still refuses absent and foreign windows", + { timeout: 240_000 }, + async () => { + const root = await createTestTempDirectory("rea-native-ui-offscreen-"); + const bundle = join(root, "Fixture.app"); + const contents = join(bundle, "Contents"); + await mkdir(join(contents, "MacOS"), { recursive: true }); + await writeFile( + join(contents, "Info.plist"), + `CFBundleExecutablefixtureCFBundleIdentifierio.rea.source-owned-offscreen-fixtureCFBundleNameREA Offscreen FixtureCFBundlePackageTypeAPPL`, + ); + const executable = join(contents, "MacOS", "fixture"); + await promisify(execFile)("/usr/bin/xcrun", [ + "swiftc", + "-module-cache-path", + join(root, "modules"), + resolve("tests/conformance/native/ui-offscreen.swift"), + "-o", + executable, + ]); + const selected = await launch(bundle, "selected"); + const foreign = await launch(bundle, "foreign"); + const target = await parseBinaryTarget(executable); + if (!target.ok) throw target.error; + + const runtime = createNativeUiHelperRuntime(process.env); + onTestFinished(() => runtime.close()); + const read = (windowId: number) => + runtime.invoke({ + pid: selected.pid, + window_id: windowId, + executable: target.value.path, + sha256: target.value.sha256, + screenshot: false, + accessibility: false, + max_nodes: 1, + }); + + expect(await read(selected.window_id)).toMatchObject({ + ok: true, + result: { window: { pid: selected.pid, window_id: selected.window_id } }, + }); + for (const windowId of [foreign.window_id, 2_147_483_000]) + expect(await read(windowId)).toMatchObject({ + ok: false, + code: "window-mismatch", + }); + + // Through the shared observation, the window is no longer called absent. Its + // outcome depends on the host's permissions, which this test does not select. + const options = { environment: process.env, runtime }; + const observed = await observeNativeUi( + target.value, + "observe_native_ui", + selected, + options, + ); + if (!observed.ok) + expect(JSON.stringify(observed.error)).not.toContain("window-mismatch"); + + // The fact the verifier's changed-bytes check reads. + const changed = await observeNativeUi( + { ...target.value, sha256: "0".repeat(64) }, + "observe_native_ui", + selected, + options, + ); + expect(changed.ok).toBe(false); + if (changed.ok) return; + expect(changed.error).toBeInstanceOf(AnalysisInputError); + expect((changed.error as AnalysisInputError).issues?.[0]).toMatchObject({ + path: ["pid"], + message: expect.stringMatching(/^target-mismatch:/u), + }); + }, +); diff --git a/tests/conformance/native/ui-offscreen.swift b/tests/conformance/native/ui-offscreen.swift new file mode 100644 index 0000000000..599a1a836f --- /dev/null +++ b/tests/conformance/native/ui-offscreen.swift @@ -0,0 +1,14 @@ +import AppKit + +// A real window that exists and is never ordered in, so it stays off screen and +// never appears on the desktop. It expires in case its launcher loses track of it. +let app = NSApplication.shared +app.setActivationPolicy(.accessory) +let window = NSWindow(contentRect: NSRect(x: 120, y: 120, width: 240, height: 80), styleMask: [.titled], backing: .buffered, defer: false) +window.title = "REA off-screen window fixture" +DispatchQueue.main.async { + let record = "{\"pid\":\(ProcessInfo.processInfo.processIdentifier),\"window_id\":\(window.windowNumber)}\n" + FileHandle.standardOutput.write(Data(record.utf8)) +} +DispatchQueue.main.asyncAfter(deadline: .now() + 300) { exit(0) } +app.run()