From b718ad166d532241fb20293e73796d492b7e2812 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:44:49 +0800 Subject: [PATCH 01/35] fix(doctor): reject unusable Kuna caches and invalid native settings (#5647) * fix(kuna): reject dangling cache roots in doctor Treat a cache path as creatable only when no directory entry occupies it. A dangling symbolic link, including one hidden by a trailing separator, fails the cache check and clears provider availability while the installation check and caller-selected registration settings remain. Fixes #5554 Fixes #5105 Co-authored-by: morluto * fix(doctor): admit the same settings native analysis rejects Project parseConfig failures into scoped engine readiness. Invalid provider preferences, startup timeouts, and other shared settings now fail the selected provider and the full audit, while a JEB-only timeout stays outside that admission. Fixes #5454 Co-authored-by: morluto * fix(doctor): report an invalid log level from doctor The dispatcher rejected REA_LOG_LEVEL before every command, so doctor never emitted its readiness check. Doctor now runs and names that setting; other commands still stop before execution. Co-authored-by: morluto * fix(doctor): require log-level failures in every scope An invalid REA_LOG_LEVEL stops the executable and MCP server before a client or skill can run, so those scopes now fail readiness too. Other native admission failures stay limited to a selected provider. Co-authored-by: morluto * fix(kuna): strip only the platform cache separator On Darwin and Linux a trailing backslash is a filename character. Doctor now removes only the selected platform separator before inspecting the cache path. Co-authored-by: morluto * fix(doctor): fail every provider on an invalid Hopper timeout REA_HOPPER_STARTUP_TIMEOUT_MS is rejected before any native engine starts. A selected non-Hopper provider now includes the existing timeout check. Co-authored-by: morluto * style(doctor): format the Hopper timeout regression Co-authored-by: morluto * fix(doctor): fail client scopes on MCP startup errors A registered client starts the MCP server, and parseConfig rejects every native admission failure before that server starts. Client readiness now includes those failures, not only REA_LOG_LEVEL. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- docs/installation.md | 7 + scripts/rea.mjs | 25 +- src/application/Doctor.test.ts | 269 +++++++++++++++++- src/application/Doctor.ts | 12 + src/application/DoctorDiagnostics.ts | 39 +++ src/application/DoctorScope.ts | 27 ++ src/cli.ts | 8 +- src/config.test.ts | 23 ++ src/config/parseConfig.ts | 31 +- src/kuna/KunaDoctor.ts | 71 ++++- src/logger.ts | 12 + .../cli/cliLogLevelConfiguration.test.ts | 61 +++- .../cli/doctorHopperDiscovery.test.ts | 69 +++++ .../providers/kuna/kunaDoctor.test.ts | 97 ++++++- .../providers/kuna/kunaSetupReadiness.test.ts | 6 +- 15 files changed, 719 insertions(+), 38 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index d18a9b6405..cbfb3e4db5 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -340,6 +340,13 @@ A scoped report has `scope.mode: "explicit"`. Its `scope_checks` determine `environment_healthy` summarizes the full audit. `--target` adds a target check, but the report remains audit-wide unless a scope option is also supplied. +An invalid `REA_LOG_LEVEL` fails every scope, including one client or the +installed skill, because the executable rejects it before any command. A +selected client also fails for every other setting that MCP startup rejects, +including `REA_GHIDRA_STARTUP_TIMEOUT_MS` and `REA_HOPPER_STARTUP_TIMEOUT_MS`. +Those settings still fail a selected provider and the audit-wide result. A +skill-only report leaves them informational. + Follow the remediation for the failed check. If a native target has several available providers, choose one with `--provider` on the CLI or `provider_id` on `open_binary`. A `capability_unavailable` failure carries diff --git a/scripts/rea.mjs b/scripts/rea.mjs index 326676a976..ab16851541 100755 --- a/scripts/rea.mjs +++ b/scripts/rea.mjs @@ -94,7 +94,9 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { const cliArgs = normalizeCliArguments(args); const outputArguments = validateCliOutputArguments(cliArgs); const configurationError = cliConfigurationError(process.env); - if (configurationError !== undefined) { + // Doctor is the readiness report for this failure. Other commands still stop + // here so they do not run with an unusable log level. + if (configurationError !== undefined && !isDoctorCommand(cliArgs)) { process.stderr.write(`${configurationError}\n`); process.exitCode = 1; } else if (!outputArguments.ok) { @@ -135,6 +137,27 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { } } +/** True when the first positional command is `doctor`, after global output flags. */ +function isDoctorCommand(arguments_) { + const valuedFlags = new Set([ + "--format", + "--filter-output", + "--token-limit", + "--token-offset", + ]); + for (let index = 0; index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === "--") return arguments_[index + 1] === "doctor"; + if (valuedFlags.has(argument)) { + index += 1; + continue; + } + if (argument.startsWith("-")) continue; + return argument === "doctor"; + } + return false; +} + /** * A reader that closes the pipe early, as `rea ... | head` does, has declined * the rest of the output. Stop writing without reporting a failure, and exit diff --git a/src/application/Doctor.test.ts b/src/application/Doctor.test.ts index b8aad49859..a7d57143d0 100644 --- a/src/application/Doctor.test.ts +++ b/src/application/Doctor.test.ts @@ -89,7 +89,7 @@ describe("doctor Hopper startup timeout", () => { }, ); - it("keeps an invalid Hopper timeout informational outside the Hopper scope", async () => { + it("keeps an invalid Hopper timeout informational when no provider is selected", async () => { const selectedHost = timeoutHost("0"); const result = await runDoctor(undefined, selectedHost, { providers: [] }); expect(result.healthy).toBe(true); @@ -100,6 +100,273 @@ describe("doctor Hopper startup timeout", () => { }), ); }); + + it("requires an invalid Hopper timeout for a ready non-Hopper provider", async () => { + const ghidra = { + providerInspections: () => + Promise.resolve([ + { + id: "ghidra" as const, + configured: true, + available: true, + providerVersion: "12.1.4", + registrationEnvironment: { GHIDRA_INSTALL_DIR: "/tools/ghidra" }, + checks: [ + { + name: "configuration", + ok: true, + code: null, + detail: "/tools/ghidra", + remediation: null, + classification: "config_drift" as const, + }, + ], + }, + ]), + }; + const blocked = await runDoctor(undefined, timeoutHost("abc", ghidra), { + providers: ["ghidra"], + }); + expect(blocked.healthy).toBe(false); + expect(blocked.scope_checks).toContainEqual( + expect.objectContaining({ + name: "hopper-startup-timeout", + ok: false, + classification: "config_drift", + }), + ); + expect( + blocked.scope_checks.filter( + ({ name }) => name === "hopper-startup-timeout", + ), + ).toHaveLength(1); + + const ready = await runDoctor(undefined, timeoutHost("120000", ghidra), { + providers: ["ghidra"], + }); + expect(ready.healthy).toBe(true); + expect(ready.scope_checks).toContainEqual( + expect.objectContaining({ name: "hopper-startup-timeout", ok: true }), + ); + }); +}); + +describe("doctor native analysis configuration", () => { + const configurationHost = ( + environment: Readonly, + ): DoctorHost => ({ + ...systemDoctorHost({ environment: { ...environment } }), + ...host({ + installationPaths: () => Promise.resolve([]), + clientRegistrations: () => Promise.resolve([]), + }), + }); + + it.each([ + ["REA_ANALYSIS_PROVIDER", "Hopper"], + ["REA_GHIDRA_STARTUP_TIMEOUT_MS", "abc"], + ["REA_SWIFTC_COMPILE_TIMEOUT_MS", "0"], + ["REA_LOG_LEVEL", "verbose"], + ["REA_GHIDRA_COMPILER_SPEC_ID", "gcc"], + ] as const)( + "rejects %s before scoped Hopper readiness reports success", + async (setting, value) => { + const result = await runDoctor( + undefined, + configurationHost({ [setting]: value }), + { providers: ["hopper"] }, + ); + + expect(result.healthy).toBe(false); + expect(result.environment_healthy).toBe(false); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: `runtime-configuration:${setting}`, + ok: false, + classification: "config_drift", + detail: expect.stringContaining(setting), + remediation: expect.stringContaining(setting), + details: { setting }, + }), + ); + }, + ); + + it("keeps a lowercase provider preference ready for the selected engine", async () => { + const result = await runDoctor( + undefined, + configurationHost({ REA_ANALYSIS_PROVIDER: "hopper" }), + { providers: ["hopper"] }, + ); + + expect(result.healthy).toBe(true); + expect( + result.checks.some(({ name }) => + name.startsWith("runtime-configuration:"), + ), + ).toBe(false); + }); + + it("keeps an invalid native setting informational when nothing starts MCP", async () => { + const result = await runDoctor( + undefined, + configurationHost({ REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc" }), + { providers: [] }, + ); + + expect(result.healthy).toBe(true); + expect(result.environment_healthy).toBe(false); + expect(result.scope_checks.map(({ name }) => name)).not.toContain( + "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ); + expect(result.informational_checks).toContainEqual( + expect.objectContaining({ + name: "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ok: false, + }), + ); + }); + + it("does not treat a JEB-only timeout as native analysis configuration", async () => { + const result = await runDoctor( + undefined, + configurationHost({ REA_JEB_MCP_TIMEOUT_MS: "abc" }), + { providers: ["hopper"] }, + ); + + expect(result.healthy).toBe(true); + expect( + result.checks.some(({ name }) => + name.startsWith("runtime-configuration:"), + ), + ).toBe(false); + }); +}); + +describe("doctor MCP startup readiness", () => { + const configurationHost = ( + environment: Readonly, + ): DoctorHost => ({ + ...systemDoctorHost({ environment: { ...environment } }), + ...host({ + installationPaths: () => Promise.resolve([]), + clientRegistrations: () => Promise.resolve([]), + }), + }); + const alignedCodex: ClientRegistrationStatus = { + client: "codex", + config_path: "/home/user/.codex/config.toml", + command: ["/current/rea", "mcp"], + state: "aligned", + remediation: null, + }; + const selectedHost = ( + environment: Readonly, + ): DoctorHost => ({ + ...configurationHost(environment), + clientRegistrations: () => Promise.resolve([alignedCodex]), + }); + + it("requires an invalid log level for a client and for the skill", async () => { + const client = await runDoctor( + undefined, + selectedHost({ + REA_LOG_LEVEL: "verbose", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc", + }), + { clients: ["codex"] }, + ); + expect(client.healthy).toBe(false); + expect(client.scope_checks).toContainEqual( + expect.objectContaining({ + name: "runtime-configuration:REA_LOG_LEVEL", + ok: false, + details: { setting: "REA_LOG_LEVEL" }, + }), + ); + expect(client.scope_checks).toContainEqual( + expect.objectContaining({ + name: "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ok: false, + }), + ); + expect(client.scope_checks).toContainEqual( + expect.objectContaining({ name: "registration:codex", ok: true }), + ); + + const skill = await runDoctor( + undefined, + selectedHost({ + REA_LOG_LEVEL: "verbose", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc", + }), + { skill: true }, + ); + expect(skill.healthy).toBe(false); + expect(skill.scope_checks).toContainEqual( + expect.objectContaining({ + name: "runtime-configuration:REA_LOG_LEVEL", + ok: false, + }), + ); + expect(skill.scope_checks.map(({ name }) => name)).not.toContain( + "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ); + }); + + it("requires every MCP startup failure for an aligned client", async () => { + const ghidraTimeout = await runDoctor( + undefined, + selectedHost({ REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc" }), + { clients: ["codex"] }, + ); + expect(ghidraTimeout.healthy).toBe(false); + expect(ghidraTimeout.scope_checks).toContainEqual( + expect.objectContaining({ + name: "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ok: false, + }), + ); + expect(ghidraTimeout.scope_checks).toContainEqual( + expect.objectContaining({ name: "registration:codex", ok: true }), + ); + + const hopperTimeout = await runDoctor( + undefined, + selectedHost({ REA_HOPPER_STARTUP_TIMEOUT_MS: "abc" }), + { clients: ["codex"] }, + ); + expect(hopperTimeout.healthy).toBe(false); + expect( + hopperTimeout.scope_checks.filter( + ({ name }) => name === "hopper-startup-timeout", + ), + ).toHaveLength(1); + + const hopperAndClient = await runDoctor( + undefined, + selectedHost({ REA_HOPPER_STARTUP_TIMEOUT_MS: "abc" }), + { clients: ["codex"], providers: ["hopper"] }, + ); + expect( + hopperAndClient.scope_checks.filter( + ({ name }) => name === "hopper-startup-timeout", + ), + ).toHaveLength(1); + + const skill = await runDoctor( + undefined, + selectedHost({ REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc" }), + { skill: true }, + ); + expect(skill.healthy).toBe(true); + expect(skill.informational_checks).toContainEqual( + expect.objectContaining({ + name: "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ok: false, + }), + ); + }); }); describe("doctor runtime check", () => { diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index f3727ac107..588c5d4acd 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -11,7 +11,10 @@ import { constants } from "node:fs"; import { access, readdir, stat } from "node:fs/promises"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import type { RetainedResourceScope } from "./AnalysisProvider.js"; +import { nativeAnalysisConfigurationError } from "../config/parseConfig.js"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; +import type { ConfigurationSettingIssue } from "../domain/configurationErrors.js"; +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { join } from "node:path"; import { analysisErrorRemediationAction } from "../domain/analysisErrorPresentation.js"; @@ -109,6 +112,11 @@ export interface DoctorHost { readonly homeDirectory: string; readonly configuredHopperPath?: string; readonly configuredHopperStartupTimeout?: string; + /** + * Settings `parseConfig` rejects before native analysis starts. + * Undefined means this host did not run that admission. + */ + readonly configurationIssues?: readonly ConfigurationSettingIssue[]; readonly configuredIlspyCmdPath?: string; macosVersion(): Promise; linuxDistribution(): Promise; @@ -508,6 +516,9 @@ export const systemDoctorHost = ( const platform = options.platform ?? process.platform; const architecture = options.architecture ?? process.arch; const environment = options.environment ?? process.env; + const configurationError = nativeAnalysisConfigurationError( + snapshotEnvironment(environment, platform), + ); const hostExecFileOutput = options.execFileOutput ?? execFileOutput; const homeDirectory = homeDirectoryFromEnvironment(environment, platform); const commandEnvironment = { env: environment }; @@ -548,6 +559,7 @@ export const systemDoctorHost = ( configuredHopperStartupTimeout: environment.REA_HOPPER_STARTUP_TIMEOUT_MS, }), + configurationIssues: configurationError?.settings ?? [], ...(environment.REA_ILSPY_CMD_PATH === undefined ? {} : { configuredIlspyCmdPath: environment.REA_ILSPY_CMD_PATH }), diff --git a/src/application/DoctorDiagnostics.ts b/src/application/DoctorDiagnostics.ts index 9f9526ad8f..f09c17889a 100644 --- a/src/application/DoctorDiagnostics.ts +++ b/src/application/DoctorDiagnostics.ts @@ -37,6 +37,7 @@ export const collectDoctorDiagnostics = async ( checks.push(hopperCheck(host, hopperPath)); const hopperTimeoutCheck = optionalHopperStartupTimeoutCheck(host); if (hopperTimeoutCheck !== undefined) checks.push(hopperTimeoutCheck); + checks.push(...runtimeConfigurationChecks(host)); const hopperVersionCheck = await optionalHopperVersionCheck(host, hopperPath); if (hopperVersionCheck !== undefined) checks.push(hopperVersionCheck); @@ -230,6 +231,44 @@ const optionalHopperStartupTimeoutCheck = ( ); }; +/** + * Project every native-analysis admission failure except the Hopper timeout, + * which already has its own check. Provider scopes include engine settings. + * Settings that block process start are marked for every doctor scope. + */ +const runtimeConfigurationChecks = ( + host: DoctorHost, +): readonly DoctorCheck[] => { + const issues = host.configurationIssues; + if (issues === undefined || issues.length === 0) return []; + const grouped: { setting: string; constraints: string[] }[] = []; + for (const issue of issues) { + if (issue.setting === "REA_HOPPER_STARTUP_TIMEOUT_MS") continue; + const existing = grouped.find((item) => item.setting === issue.setting); + if (existing === undefined) + grouped.push({ setting: issue.setting, constraints: [issue.constraint] }); + else existing.constraints.push(issue.constraint); + } + return grouped.map(({ setting, constraints }) => ({ + ...check( + `runtime-configuration:${setting}`, + false, + constraints + .map((constraint) => + constraint.includes(setting) + ? constraint.replace(/\.$/u, "") + : `${setting}: ${constraint}`, + ) + .join("; "), + { + classification: "config_drift", + remediation: `Correct ${setting} in the environment that starts REA (your shell or the MCP client's registration), then rerun.`, + }, + ), + details: { setting }, + })); +}; + const optionalHopperVersionCheck = async ( host: DoctorHost, hopperPath: string | undefined, diff --git a/src/application/DoctorScope.ts b/src/application/DoctorScope.ts index da22d43b41..49df606bab 100644 --- a/src/application/DoctorScope.ts +++ b/src/application/DoctorScope.ts @@ -1,4 +1,5 @@ import { uniqueSorted } from "../domain/canonicalOrdering.js"; +import { blocksProcessStart } from "../logger.js"; import type { ClientRegistrationStatus, UnhealthyClientRegistrationStatus, @@ -80,8 +81,33 @@ export const scopeDoctorChecks = ( const hopperHostChecks = input.scope.providers.includes("hopper") ? input.checks.filter(({ name }) => name === "host") : []; + // parseConfig rejects these before the MCP server starts. Hopper's own + // scope already includes hopper-startup-timeout through its provider checks. + const blocksMcpStartup = (check: DoctorCheck): boolean => + check.name.startsWith("runtime-configuration:") || + check.name === "hopper-startup-timeout"; + const blocksProcess = (check: DoctorCheck): boolean => { + const setting = check.details?.setting; + return ( + check.name.startsWith("runtime-configuration:") && + typeof setting === "string" && + blocksProcessStart(setting) + ); + }; + const exercisesMcpStartup = + input.scope.clients.length > 0 || input.scope.providers.length > 0; + const startupChecks = ( + exercisesMcpStartup + ? input.checks.filter(blocksMcpStartup) + : input.checks.filter(blocksProcess) + ).filter( + (check) => + check.name !== "hopper-startup-timeout" || + !input.scope.providers.includes("hopper"), + ); const scopeChecks = [ ...coreChecks, + ...startupChecks, ...hopperHostChecks, ...selectedProviderChecks, ...clientChecks, @@ -91,6 +117,7 @@ export const scopeDoctorChecks = ( return { healthy: coreChecks.every(({ ok }) => ok) && + startupChecks.every(({ ok }) => ok) && hopperHostChecks.every(({ ok }) => ok) && clientChecks.every(({ ok }) => ok) && skillChecks.every(({ ok }) => ok) && diff --git a/src/cli.ts b/src/cli.ts index df2c15c15f..d7e9e72ad9 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -58,7 +58,10 @@ import { import { registerReverseEngineeringCommands } from "./cli/reverseEngineeringCommands.js"; import { registerCutterCommands } from "./cli/cutterCommands.js"; -/** Describe CLI settings that are invalid before any command runs, as MCP does. */ +/** + * Describe CLI settings that are invalid before any command runs, as MCP does. + * Keep this aligned with PROCESS_START_CONFIGURATION_SETTINGS. + */ export const cliConfigurationError = ( selectedEnvironment: Readonly>, ): string | undefined => { @@ -82,7 +85,8 @@ export const createCli = ( ): CliInstance => { const environment = snapshotEnvironment(selectedEnvironment); const analysis = createDirectAnalysis(environment); - // The executable reports an invalid level through cliConfigurationError. + // The executable reports an invalid level before every command except doctor. + // Doctor keeps going so its readiness report can name REA_LOG_LEVEL. const level = environment.REA_LOG_LEVEL === undefined ? undefined diff --git a/src/config.test.ts b/src/config.test.ts index d85ecb76cb..1c2017017a 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { defaultHopperLauncherPath, + nativeAnalysisConfigurationError, parseConfig, } from "./config/parseConfig.js"; import { @@ -98,6 +99,28 @@ describe("runtime configuration", () => { expect(parseConfig({ REA_ANALYSIS_PROVIDER: invalid }).ok).toBe(false); }); + it("shares native admission with doctor and ignores the JEB-only timeout", () => { + expect(nativeAnalysisConfigurationError({})).toBeUndefined(); + expect( + nativeAnalysisConfigurationError({ REA_JEB_MCP_TIMEOUT_MS: "abc" }), + ).toBeUndefined(); + expect(parseConfig({ REA_JEB_MCP_TIMEOUT_MS: "abc" }).ok).toBe(true); + const rejected = nativeAnalysisConfigurationError({ + REA_ANALYSIS_PROVIDER: "Hopper", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc", + }); + expect(rejected?.settings.map(({ setting }) => setting)).toEqual([ + "REA_ANALYSIS_PROVIDER", + "REA_GHIDRA_STARTUP_TIMEOUT_MS", + ]); + expect( + parseConfig({ + REA_ANALYSIS_PROVIDER: "Hopper", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc", + }).ok, + ).toBe(false); + }); + it("parses absolute BYO Ghidra and optional Java paths", () => { expect( parseConfig({ diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index c00f787a8d..8fa259d682 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -39,16 +39,35 @@ export const defaultHopperLauncherPath = ( ); }; +/** + * Environment native analysis admits. JEB validates its own timeout during + * optional provider composition; that setting must not block unrelated + * CLI/MCP startup. Registration validation still checks it directly. + */ +const nativeAnalysisEnvironment = ( + environment: Readonly>, +): Readonly> => { + const { REA_JEB_MCP_TIMEOUT_MS: _jebTimeout, ...appEnvironment } = + environment; + return appEnvironment; +}; + +/** + * The configuration error native commands return, without probing launchers. + * Doctor uses this so readiness and analysis admit the same settings. + */ +export const nativeAnalysisConfigurationError = ( + environment: Readonly>, +): ConfigurationError | undefined => { + const parsed = parseEnvironment(nativeAnalysisEnvironment(environment)); + return parsed.ok ? undefined : parsed.error; +}; + /** Parse provider configuration and runtime prerequisites at the composition root. */ export const parseConfig = ( environment: Readonly>, ): Result => { - // JEB validates its own timeout during optional provider composition. It is - // not an AppConfig setting and must not prevent unrelated CLI/MCP startup. - // Registration validation still checks the complete environment directly. - const { REA_JEB_MCP_TIMEOUT_MS: _jebTimeout, ...appEnvironment } = - environment; - const parsed = parseEnvironment(appEnvironment); + const parsed = parseEnvironment(nativeAnalysisEnvironment(environment)); if (!parsed.ok) return parsed; const env = parsed.value; return ok({ diff --git a/src/kuna/KunaDoctor.ts b/src/kuna/KunaDoctor.ts index b5fbfe43a6..d4a533ab72 100644 --- a/src/kuna/KunaDoctor.ts +++ b/src/kuna/KunaDoctor.ts @@ -1,6 +1,6 @@ import { constants } from "node:fs"; -import { access, stat } from "node:fs/promises"; -import { dirname } from "node:path"; +import { access, lstat, stat } from "node:fs/promises"; +import { dirname, posix, win32 } from "node:path"; import type { DoctorProviderCheck, @@ -151,10 +151,11 @@ export const inspectKunaProvider = async ( }; } const { value } = installation; + const cache = await cacheCheck(environment, platform); return { id: KUNA_PROVIDER_ID, configured, - available: true, + available: cache.ok, providerVersion: value.version, registrationEnvironment: Object.fromEntries( REGISTRATION_SETTINGS.flatMap((name) => { @@ -181,7 +182,7 @@ export const inspectKunaProvider = async ( sha256: value.sha256, }, }, - await cacheCheck(environment, platform), + cache, ], }; }; @@ -226,8 +227,10 @@ const cacheCheck = async ( }; const { cacheRoot, cacheMaxBytes } = configuration.value; const details = { cache_root: cacheRoot, cache_max_bytes: cacheMaxBytes }; + // A trailing separator makes lstat follow a dangling link and report ENOENT. + const inspected = cacheRootWithoutTrailingSeparator(cacheRoot, platform); try { - if (!(await stat(cacheRoot)).isDirectory()) + if (!(await stat(inspected)).isDirectory()) return { name: "cache", ok: false, @@ -238,10 +241,10 @@ const cacheCheck = async ( classification: "config_drift", details, }; - await access(cacheRoot, constants.W_OK | constants.X_OK); + await access(inspected, constants.W_OK | constants.X_OK); } catch (cause: unknown) { - if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") { - const creatable = await nearestDirectoryWritable(cacheRoot); + if (isMissing(cause)) { + const creatable = await nearestDirectoryWritable(inspected); return creatable.ok ? { name: "cache", @@ -285,13 +288,18 @@ const cacheCheck = async ( }; }; -/** Whether the nearest existing ancestor is a directory REA may create entries in. */ +/** + * Whether the nearest existing entry on `path` is a directory REA may create + * entries in. A missing target is only absent when no entry occupies its name; + * a dangling symbolic link blocks creation just as a regular file does. + * `path` must not end in a separator, or lstat follows a dangling link. + */ const nearestDirectoryWritable = async ( path: string, ): Promise< { readonly ok: true } | { readonly ok: false; readonly reason: string } > => { - let candidate = dirname(path); + let candidate = path; for (;;) { try { if (!(await stat(candidate)).isDirectory()) @@ -299,15 +307,50 @@ const nearestDirectoryWritable = async ( await access(candidate, constants.W_OK | constants.X_OK); return { ok: true }; } catch (cause: unknown) { - const missing = - cause instanceof Error && "code" in cause && cause.code === "ENOENT"; const parent = dirname(candidate); - if (!missing || parent === candidate) + const absent = isMissing(cause) ? await entryAbsent(candidate) : false; + if (absent !== true || parent === candidate) return { ok: false, - reason: `${candidate} is not writable: ${cause instanceof Error ? cause.message : String(cause)}`, + reason: + typeof absent === "string" + ? absent + : `${candidate} is not writable: ${cause instanceof Error ? cause.message : String(cause)}`, }; candidate = parent; } } }; + +const isMissing = (cause: unknown): boolean => + cause instanceof Error && "code" in cause && cause.code === "ENOENT"; + +/** True when nothing occupies `path`; otherwise why its name is unusable. */ +const entryAbsent = async (path: string): Promise => { + try { + return (await lstat(path)).isSymbolicLink() + ? `${path} is a symbolic link whose target does not exist.` + : `${path} exists but cannot be resolved.`; + } catch (cause: unknown) { + return isMissing(cause) + ? true + : `${path} cannot be inspected: ${cause instanceof Error ? cause.message : String(cause)}`; + } +}; + +/** + * Drop a terminal separator without turning a filesystem root into its parent. + * On Darwin and Linux, `\` is a filename character, so only `/` is removed. + */ +const cacheRootWithoutTrailingSeparator = ( + cacheRoot: string, + platform: NodeJS.Platform, +): string => { + const pathApi = platform === "win32" ? win32 : posix; + const root = pathApi.parse(cacheRoot).root; + const separator = pathApi.sep.charCodeAt(0); + let end = cacheRoot.length; + while (end > root.length && cacheRoot.charCodeAt(end - 1) === separator) + end -= 1; + return end === cacheRoot.length ? cacheRoot : cacheRoot.slice(0, end); +}; diff --git a/src/logger.ts b/src/logger.ts index 05fc3822fe..6e5280cd8e 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -4,6 +4,18 @@ import { z } from "zod"; import { ConfigurationError } from "./domain/configurationErrors.js"; import { err, ok, type Result } from "./domain/result.js"; +/** + * Settings the executable rejects before a command starts. MCP startup rejects + * the same values. Doctor requires them in every readiness scope. + */ +export const PROCESS_START_CONFIGURATION_SETTINGS = ["REA_LOG_LEVEL"] as const; + +/** True when `setting` prevents the REA process from starting. */ +export const blocksProcessStart = (setting: string): boolean => + PROCESS_START_CONFIGURATION_SETTINGS.some( + (candidate) => candidate === setting, + ); + /** Supported structured log levels at the process boundary. */ export type LogLevel = z.infer; diff --git a/tests/boundary/cli/cliLogLevelConfiguration.test.ts b/tests/boundary/cli/cliLogLevelConfiguration.test.ts index cc09a3b944..fb6490091f 100644 --- a/tests/boundary/cli/cliLogLevelConfiguration.test.ts +++ b/tests/boundary/cli/cliLogLevelConfiguration.test.ts @@ -2,20 +2,59 @@ import { expect } from "vitest"; import { cliTest } from "../../support/cli/cliFixture.js"; +const invalidLogLevelMessage = + 'REA configuration is invalid: REA_LOG_LEVEL: Invalid option: expected one of "trace"|"debug"|"info"|"warn"|"error"|"fatal"|"silent".\n'; + for (const level of ["loud", "INFO"]) cliTest( `reports REA_LOG_LEVEL=${level} as invalid configuration`, async ({ cli }) => { - for (const arguments_ of [["doctor", "--json"], ["--help"]]) { - const result = await cli.run({ - arguments: arguments_, - environment: { REA_LOG_LEVEL: level }, - }); - expect(result.exitCode).toBe(1); - expect(result.stdout).toBe(""); - expect(result.stderr).toBe( - 'REA configuration is invalid: REA_LOG_LEVEL: Invalid option: expected one of "trace"|"debug"|"info"|"warn"|"error"|"fatal"|"silent".\n', - ); - } + const help = await cli.run({ + arguments: ["--help"], + environment: { REA_LOG_LEVEL: level }, + }); + expect(help.exitCode).toBe(1); + expect(help.stdout).toBe(""); + expect(help.stderr).toBe(invalidLogLevelMessage); + + const doctor = await cli.run({ + arguments: ["--json", "doctor", "--provider", "hopper"], + environment: { REA_LOG_LEVEL: level }, + }); + expect(doctor.exitCode, doctor.stderr).toBe(1); + expect(doctor.stderr).toBe(""); + expect(doctor.json).toMatchObject({ + healthy: false, + scope_checks: expect.arrayContaining([ + expect.objectContaining({ + name: "runtime-configuration:REA_LOG_LEVEL", + ok: false, + classification: "config_drift", + detail: expect.stringContaining("REA_LOG_LEVEL"), + }), + ]), + }); + + const client = await cli.run({ + arguments: ["--json", "doctor", "--client", "codex"], + environment: { REA_LOG_LEVEL: level }, + }); + expect(client.exitCode, client.stderr).toBe(1); + expect(client.stderr).toBe(""); + expect(client.json).toMatchObject({ + healthy: false, + scope: { mode: "explicit", clients: ["codex"], providers: [] }, + scope_checks: expect.arrayContaining([ + expect.objectContaining({ + name: "runtime-configuration:REA_LOG_LEVEL", + ok: false, + }), + ]), + informational_checks: expect.not.arrayContaining([ + expect.objectContaining({ + name: "runtime-configuration:REA_LOG_LEVEL", + }), + ]), + }); }, ); diff --git a/tests/boundary/cli/doctorHopperDiscovery.test.ts b/tests/boundary/cli/doctorHopperDiscovery.test.ts index 1a33db49cd..c613dfd2d8 100644 --- a/tests/boundary/cli/doctorHopperDiscovery.test.ts +++ b/tests/boundary/cli/doctorHopperDiscovery.test.ts @@ -60,6 +60,75 @@ cliTest.skipIf(!["darwin", "linux"].includes(process.platform))( }, ); +cliTest.skipIf(!["darwin", "linux"].includes(process.platform))( + "reports native configuration that blocks the selected engine", + async ({ cli }) => { + const home = await createTestTempDirectory("rea-doctor-native-config-"); + const shared = { + HOME: home, + USERPROFILE: home, + GHIDRA_INSTALL_DIR: join(home, "missing-ghidra"), + HOPPER_LAUNCHER_PATH: join(home, "missing-hopper"), + }; + const invalid = await cli.run({ + arguments: ["doctor", "--provider", "hopper", "--json"], + environment: { + ...shared, + REA_ANALYSIS_PROVIDER: "Hopper", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc", + }, + }); + expect(invalid.exitCode, invalid.stderr).toBe(1); + expect(invalid.json).toMatchObject({ + healthy: false, + scope_checks: expect.arrayContaining([ + expect.objectContaining({ + name: "runtime-configuration:REA_ANALYSIS_PROVIDER", + ok: false, + classification: "config_drift", + }), + expect.objectContaining({ + name: "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ok: false, + classification: "config_drift", + }), + ]), + }); + + const jebOnly = await cli.run({ + arguments: ["doctor", "--provider", "hopper", "--json"], + environment: { ...shared, REA_JEB_MCP_TIMEOUT_MS: "abc" }, + }); + const checks = (jebOnly.json as { checks?: { name: string }[] } | undefined) + ?.checks; + expect( + checks?.some((check) => check.name.startsWith("runtime-configuration:")), + ).toBe(false); + }, +); + +cliTest( + "reports MCP startup failures for a selected client", + async ({ cli }) => { + const doctor = await cli.run({ + arguments: ["--json", "doctor", "--client", "codex"], + environment: { REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc" }, + }); + expect(doctor.exitCode, doctor.stderr).toBe(1); + expect(doctor.stderr).toBe(""); + expect(doctor.json).toMatchObject({ + healthy: false, + scope: { mode: "explicit", clients: ["codex"], providers: [] }, + scope_checks: expect.arrayContaining([ + expect.objectContaining({ + name: "runtime-configuration:REA_GHIDRA_STARTUP_TIMEOUT_MS", + ok: false, + }), + ]), + }); + }, +); + cliTest.skipIf(!["darwin", "linux"].includes(process.platform))( "retains a configured Hopper path in host diagnostics without launching it", async ({ cli }) => { diff --git a/tests/boundary/providers/kuna/kunaDoctor.test.ts b/tests/boundary/providers/kuna/kunaDoctor.test.ts index 758b94d570..5a3acd0aef 100644 --- a/tests/boundary/providers/kuna/kunaDoctor.test.ts +++ b/tests/boundary/providers/kuna/kunaDoctor.test.ts @@ -1,5 +1,5 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { lstat, mkdir, readlink, symlink, writeFile } from "node:fs/promises"; +import { join, sep as separator } from "node:path"; import { expect, it as test } from "vitest"; @@ -97,11 +97,17 @@ it("reports a cache root REA cannot use without creating anything", async () => REA_KUNA_CACHE_DIR: existing, }); + expect(blocked.available).toBe(false); + expect(blocked.checks[0]).toMatchObject({ name: "installation", ok: true }); expect(blocked.checks[1]).toMatchObject({ name: "cache", ok: false, code: "open_options_invalid", }); + expect(blocked.registrationEnvironment).toMatchObject({ + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: occupied, + }); expect(ready.checks[1]).toMatchObject({ name: "cache", ok: true, @@ -139,6 +145,93 @@ it("reports a cache root REA cannot create under its nearest existing parent", a }); }); +it("reports a dangling cache link as unusable and accepts a resolving link", async () => { + const root = await createTestTempDirectory("rea-kuna-doctor-link-"); + const kuna = await writeFakeKuna(root); + const dangling = join(root, "dangling"); + const missing = join(root, "missing-target"); + await symlink(missing, dangling); + const target = join(root, "target"); + await mkdir(target); + const resolving = join(root, "resolving"); + await symlink(target, resolving); + + const direct = await inspectKunaProvider({ + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: dangling, + }); + const trailing = await inspectKunaProvider({ + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: `${dangling}${separator}`, + }); + const beneath = await inspectKunaProvider({ + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: join(dangling, "nested", "cache"), + }); + const ready = await inspectKunaProvider({ + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: resolving, + }); + const readyTrailing = await inspectKunaProvider({ + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: `${resolving}${separator}`, + }); + + for (const blocked of [direct, trailing, beneath]) { + expect(blocked.available).toBe(false); + expect(blocked.checks[1]).toMatchObject({ + name: "cache", + ok: false, + code: "open_options_invalid", + detail: expect.stringContaining( + `${dangling} is a symbolic link whose target does not exist.`, + ), + }); + } + expect(trailing.checks[1]).toMatchObject({ + detail: expect.stringContaining(`${dangling}${separator}`), + details: { cache_root: `${dangling}${separator}` }, + }); + for (const usable of [ready, readyTrailing]) { + expect(usable.available).toBe(true); + expect(usable.checks[1]).toMatchObject({ + name: "cache", + ok: true, + detail: expect.stringContaining("cached in"), + }); + } + expect(ready.checks[1]?.detail).toContain(resolving); + // Diagnosis neither creates the cache nor repairs the link. + expect(await readlink(dangling)).toBe(missing); + await expect(lstat(missing)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(lstat(join(dangling, "nested"))).rejects.toThrow(); +}); + +it("keeps a trailing backslash as part of a POSIX cache path", async () => { + const root = await createTestTempDirectory("rea-kuna-doctor-backslash-"); + const kuna = await writeFakeKuna(root); + const directory = join(root, "cache"); + await mkdir(directory); + const named = `${directory}\\`; + await writeFile(named, "not-a-directory"); + + const inspection = await inspectKunaProvider({ + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: named, + }); + + expect(inspection.available).toBe(false); + expect(inspection.checks[1]).toMatchObject({ + name: "cache", + ok: false, + code: "open_options_invalid", + detail: expect.stringContaining(`${named} exists but is not a directory.`), + details: expect.objectContaining({ cache_root: named }), + }); + expect((await lstat(directory)).isDirectory()).toBe(true); + expect((await lstat(named)).isFile()).toBe(true); +}); + const doctorEnvironment = (root: string, kuna: string): NodeJS.ProcessEnv => ({ HOME: root, PATH: "", diff --git a/tests/boundary/providers/kuna/kunaSetupReadiness.test.ts b/tests/boundary/providers/kuna/kunaSetupReadiness.test.ts index 95ca8eb2e6..9367c6e987 100644 --- a/tests/boundary/providers/kuna/kunaSetupReadiness.test.ts +++ b/tests/boundary/providers/kuna/kunaSetupReadiness.test.ts @@ -34,7 +34,11 @@ test.each([ REA_KUNA_COMMAND: command, REA_KUNA_CACHE_DIR: cachePath, }); - expect(inspection.available).toBe(true); + expect(inspection.available).toBe(cache === "ready"); + expect(inspection.checks[0]).toMatchObject({ + name: "installation", + ok: true, + }); expect(inspection.checks[1]).toMatchObject({ name: "cache", ok: cache === "ready", From 53e0943223b016c5e1782aa76f3ec2297dc4313b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:45:10 +0800 Subject: [PATCH 02/35] fix(javascript): preserve path reasons, module coverage, and property provenance (#5651) * fix(javascript): preserve selected path host resolution reasons Classify ELOOP and ENAMETOOLONG while resolving the caller-selected input so the path and host code stay on the invalid-request issue. Refs #5623 Co-authored-by: morluto * fix(javascript): count summary coverage in module units Use the module population for both examined and total counts. Non-module graph nodes remain in the application coverage section. Refs #5464 Co-authored-by: morluto * fix(javascript): connect direct property reads to return and argument flow Record the unwrapped value expression and alias the object projection's read occurrence so a direct member read stays in backward provenance. Refs #5585, #5648 Co-authored-by: morluto * fix(javascript): count return wrappers against expression depth Evaluate the original return expression so erased TypeScript and Flow wrappers stay inside the expression-depth budget. Keep the unwrapped node for identity and value ranges. Co-authored-by: morluto * fix(javascript): describe ELOOP as unresolved symlink traversal A host can return ELOOP for a cycle or for a chain past its traversal limit. Keep the selected path and code without asserting that a cycle exists. Co-authored-by: morluto * test(javascript): skip symlink traversal fixtures on Windows Keep the overlong-name case portable. Windows does not produce ELOOP for these fixtures, matching the other path-failure tests. Co-authored-by: morluto * fix(javascript): describe ENAMETOOLONG as a path or component limit realpath reports that code for a long component and for a long full path. Keep the selected path and host code without claiming only a filename was too long. Co-authored-by: morluto * style(javascript): format return-depth and object-read changes oxfmt rejected the return-depth test and the object-read index. No behavior change. Co-authored-by: morluto * test(ida): expect unchanged invalid requests to be nonretryable The shared projector no longer marks invalid_request retryable. This lookup assertion was not updated with that contract. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- .../JavaScriptArtifactReconstruction.ts | 21 +++++- .../JavaScriptSemanticGraphBuilder.test.ts | 50 +++++++++++++ .../JavaScriptSemanticGraphBuilder.ts | 10 +++ .../JavaScriptSemanticGraphFlowProjection.ts | 61 ++++++++++++++++ src/domain/analysisView/analysisView.test.ts | 48 +++++++++++++ .../analysisView/javascriptApplicationView.ts | 4 +- .../javascriptSemanticAnalysis.unions.test.ts | 31 ++++++++ .../javascript/javascriptSemanticCalls.ts | 10 ++- src/domain/javascript/javascriptSemanticIr.ts | 7 ++ .../javascript/javascriptSemanticReturns.ts | 9 ++- .../JavaScriptAnalysisWorkerSchemas.ts | 1 + ...avascriptApplicationInputSelection.test.ts | 70 ++++++++++++++++++- 12 files changed, 315 insertions(+), 7 deletions(-) diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index b25c545a8a..1a2befbd9c 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -343,8 +343,9 @@ export const reconstructJavaScriptArtifact = async ( const OPERATION = "analyze_javascript_application"; /** - * Resolve the caller-selected input. A missing or unreadable selection is a - * caller or host-permission failure, not a damaged artifact. + * Resolve the caller-selected input. A missing, looping, overlong, or + * unreadable selection is a caller or host-resolution failure, not a damaged + * artifact. */ const resolveSelectedInput = async (inputPath: string): Promise => { try { @@ -366,10 +367,26 @@ const resolveSelectedInput = async (inputPath: string): Promise => { message: `No file or directory exists at the selected input path (${code}): ${inputPath}`, }, ]); + if (code === "ELOOP" || code === "ENAMETOOLONG") + throw new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["input_path"], + reason: "invalid_value", + message: selectedInputResolutionMessage(code, inputPath), + }, + ]); throw cause; } }; +const selectedInputResolutionMessage = ( + code: "ELOOP" | "ENAMETOOLONG", + inputPath: string, +): string => + code === "ENAMETOOLONG" + ? `Selected input path exceeds the host path or component length limit (ENAMETOOLONG): ${inputPath}` + : `Selected input path could not be resolved because symbolic-link traversal looped or exceeded the host limit (ELOOP): ${inputPath}`; + /** The selected input's kind, checked against the caller's requested format. */ const resolveFormat = async ( path: string, diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts index c3c4fea0fc..ecf70049da 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts @@ -172,6 +172,56 @@ it("continues to project an ordinary member access as a property read", () => { ); }); +it("keeps a directly returned or passed property in backward provenance", () => { + const source = ` + const settings = { index: "products" }; + function viaLocal() { + const value = settings.index; + return value; + } + function viaMember() { + return settings.index; + } + function viaParen() { + return (settings.index); + } + const arrow = () => settings.index; + function take(value) { return value; } + function added() { return settings.index + "!"; } + const a = viaLocal(); + const b = viaMember(); + const direct = settings.index; + const c = viaParen(); + const d = arrow(); + const e = take(settings.index); + const g = take((settings.index)); + const kept = settings.index; + const local = take(kept); + const concatenated = added(); + `; + const graph = graphFor(source); + const provenance = (binding: string, candidates: boolean) => { + const node = graph.nodes.find( + (item) => item.kind === "binding" && item.label === binding, + ); + if (node === undefined) throw new Error(`missing ${binding}`); + const trace = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: node.node_id }, + direction: "backward-provenance", + include_ambiguous_dynamic_edges: candidates, + }); + return trace.nodes.some( + (item) => item.kind === "literal" && item.properties.value === "products", + ); + }; + for (const binding of ["a", "b", "direct", "c", "d", "e", "g", "local"]) { + expect(provenance(binding, false), binding).toBe(true); + expect(provenance(binding, true), `${binding} candidates`).toBe(true); + } + expect(provenance("concatenated", false)).toBe(false); + expect(provenance("concatenated", true)).toBe(false); +}); + it("keeps dynamic calls explicit and returns complete deterministic results", () => { const graph = graphFor(` const handlers = { ready() { return 1; } }; diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 3417fbb895..a890d54225 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -60,6 +60,7 @@ import { projectSemanticFrontiers, projectSemanticPromises, projectSemanticReturnValues, + semanticObjectReadOccurrence, type SemanticFlowProjectionContext, } from "./JavaScriptSemanticGraphFlowProjection.js"; import { @@ -828,6 +829,15 @@ const projectCalls = (context: FileContext): void => { ? "resolved" : "candidate", }); + addRelation(context.state, { + source: semanticObjectReadOccurrence( + context, + argument.valueExpressionLocation, + ), + target: argumentNode, + relation: "aliases", + resolution: "resolved", + }); } } for (const flow of context.ir.argumentFlows) { diff --git a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts index 052a132618..59ab8b52de 100644 --- a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts @@ -1,4 +1,5 @@ import type { JavaScriptSemanticGraphNode } from "../../domain/javascript/javascriptSemanticGraphSchemas.js"; +import { javaScriptSemanticNodeId } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; import { sourceRangesEqual } from "../../domain/javascript/javascriptStaticAnalysisHelpers.js"; import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; @@ -190,6 +191,57 @@ const addPromiseUnknown = ( }); }; +interface ObjectReadIndex { + readonly state: SemanticGraphProjectionState; + readonly nodes: ReadonlyMap; +} + +const objectReadIndexes = new WeakMap(); + +const objectReadRangeKey = (location: JavaScriptSourceRange): string => + `${String(location.start.line)}:${String(location.start.column)}:${String(location.end.line)}:${String(location.end.column)}`; + +/** + * The object projection owns a member read's occurrence. Index it once per + * file so return and argument flow can connect that expression without + * scanning every property read again. + */ +const objectReadOccurrences = ( + context: SemanticFlowProjectionContext, +): ReadonlyMap => { + const cached = objectReadIndexes.get(context.ir); + if (cached !== undefined && cached.state === context.state) + return cached.nodes; + const nodes = new Map(); + for (const operation of context.ir.objectOperations) { + if (operation.kind !== "read") continue; + const node = context.state.nodes.get( + javaScriptSemanticNodeId({ + kind: "expression", + identity: { + artifact_sha256: context.file.sha256, + module_path: context.file.path, + source_range: operation.location, + role_key: operation.objectOperationId, + }, + }), + ); + if (node !== undefined) + nodes.set(objectReadRangeKey(operation.location), node); + } + objectReadIndexes.set(context.ir, { state: context.state, nodes }); + return nodes; +}; + +/** Return the retained read occurrence for one unwrapped member expression. */ +export const semanticObjectReadOccurrence = ( + context: SemanticFlowProjectionContext, + location: JavaScriptSourceRange | null, +): JavaScriptSemanticGraphNode | undefined => + location === null + ? undefined + : objectReadOccurrences(context).get(objectReadRangeKey(location)); + /** Link direct return expressions to their retained semantic references. */ export const projectSemanticReturnValues = ( context: SemanticFlowProjectionContext, @@ -212,6 +264,15 @@ export const projectSemanticReturnValues = ( ? "resolved" : "candidate", }); + addSemanticGraphRelation(context.state, { + source: semanticObjectReadOccurrence( + context, + site.valueExpressionLocation, + ), + target: returnNode, + relation: "aliases", + resolution: "resolved", + }); } }; diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index 87d1930f19..6e79299724 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -429,6 +429,54 @@ it("rejects ambiguous layout names with candidate indexes", () => { }); }); +it("counts JavaScript summary coverage in module units", () => { + const analysis = analysisViewJavaScriptAnalysisWithSource(); + const moduleNode = analysis.graph.nodes[0]; + const observation = moduleNode?.observations[0]; + if (moduleNode === undefined || observation === undefined) + throw new Error("missing module observation"); + const packageNode = createJavaScriptApplicationNode({ + kind: "artifact", + identity: moduleNode.identity, + observations: moduleNode.observations.map((item) => ({ + label: item.label, + properties: item.properties, + evidence: item.evidence, + ...(item.source_map_reference === null + ? {} + : { source_map_reference: item.source_map_reference }), + })), + }); + const graph = createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [moduleNode.node_id], + nodes: [moduleNode, packageNode], + edges: [], + coverage: analysis.graph.coverage, + limitations: analysis.graph.limitations, + }); + const parent = javascriptParent( + analysisViewBindJavaScriptGraphs(analysis, graph), + ); + const summary = projectAnalysisView(parent, { kind: "summary" }); + if (!summary.ok) throw summary.error; + expect(summary.value.coverage).toEqual({ + status: "complete-within-view", + examined: 1, + total: 1, + next_offset: null, + exhausted: true, + }); + const page = projectAnalysisView(parent, { + kind: "page", + collection: "modules", + offset: 0, + limit: 8, + }); + if (!page.ok) throw page.error; + expect(page.value.coverage).toMatchObject({ examined: 1, total: 1 }); +}); + it("projects JavaScript summary and preserves selected module observations", () => { const analysis = analysisViewJavaScriptAnalysisWithSource(); const parent = javascriptParent(analysis); diff --git a/src/domain/analysisView/javascriptApplicationView.ts b/src/domain/analysisView/javascriptApplicationView.ts index ca9d96b76e..13490df699 100644 --- a/src/domain/analysisView/javascriptApplicationView.ts +++ b/src/domain/analysisView/javascriptApplicationView.ts @@ -350,6 +350,8 @@ export const projectJavaScriptApplicationView = ( }); } if (view.kind === "summary") { + // Summary coverage counts modules. Graph nodes that are not modules stay + // in summary.coverage.application rather than inflating this total. const moduleCount = analysis.graph.nodes.reduce( (count, node) => count + Number(isModuleNode(node)), 0, @@ -384,7 +386,7 @@ export const projectJavaScriptApplicationView = ( }), limitation_count: analysis.limitations.length, }, - coverage: completeCoverage(moduleCount, analysis.graph.nodes.length), + coverage: completeCoverage(moduleCount, moduleCount), ...shared, }); } diff --git a/src/domain/javascript/javascriptSemanticAnalysis.unions.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.unions.test.ts index cfd7da8795..eebaf98daf 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.unions.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.unions.test.ts @@ -503,6 +503,37 @@ describe("semantic expression depth bounds", () => { }); }); + it("counts erased TypeScript wrappers on a return against expression depth", () => { + const source = "function answer() { return true; }"; + const parsed = parseJavaScriptSource(source); + const declaration = parsed?.program.body[0]; + const statement = t.isFunctionDeclaration(declaration) + ? declaration.body.body[0] + : undefined; + const argument = t.isReturnStatement(statement) + ? statement.argument + : undefined; + if (parsed === null || !t.isReturnStatement(statement) || argument == null) + throw new Error("Expected a returned value"); + let expression = argument; + for (let index = 0; index < 300; index += 1) + expression = t.tsAsExpression(expression, t.tsBooleanKeyword()); + statement.argument = expression; + + const ir = analyzeParsedJavaScriptSemantics(parsed); + const site = ir.callables.find((item) => item.name === "answer") + ?.returnSites[0]; + expect(site?.value).toMatchObject({ + status: "unknown", + resourceLimit: "expression-depth", + }); + const literalStart = source.indexOf("true"); + expect(site?.valueExpressionLocation).toEqual({ + start: { line: 1, column: literalStart }, + end: { line: 1, column: literalStart + "true".length }, + }); + }); + it("counts erased TypeScript wrappers against expression depth", () => { const parsed = parseJavaScriptSource("const answer = true;"); const declaration = parsed?.program.body[0]; diff --git a/src/domain/javascript/javascriptSemanticCalls.ts b/src/domain/javascript/javascriptSemanticCalls.ts index 65c342fefb..3abe4579a6 100644 --- a/src/domain/javascript/javascriptSemanticCalls.ts +++ b/src/domain/javascript/javascriptSemanticCalls.ts @@ -11,7 +11,10 @@ import type { } from "./javascriptSemanticIr.js"; import { semanticReferenceRole } from "./javascriptSemanticProjection.js"; import { createSemanticCallableStack } from "./javascriptSemanticCallableStack.js"; -import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; +import { + semanticStaticPropertyKey, + unwrapJavaScriptExpression, +} from "./javascriptAstValues.js"; import { resolveSemanticBindingState, isUnshadowedGlobal, @@ -210,9 +213,14 @@ const retainedArguments = ( for (const [index, node] of nodes.entries()) { // Babel recovery can leave null slots; retain the other argument positions. if (node === null) continue; + const valueNode = + t.isExpression(node) && !t.isSpreadElement(node) + ? unwrapJavaScriptExpression(node).node + : null; retained.push({ index, location: range(node), + valueExpressionLocation: valueNode === null ? null : range(valueNode), spread: t.isSpreadElement(node), }); } diff --git a/src/domain/javascript/javascriptSemanticIr.ts b/src/domain/javascript/javascriptSemanticIr.ts index 5355b3ad33..f153ea39ad 100644 --- a/src/domain/javascript/javascriptSemanticIr.ts +++ b/src/domain/javascript/javascriptSemanticIr.ts @@ -109,6 +109,11 @@ export interface JavaScriptSemanticReturnSite { readonly returnSiteId: string; readonly location: JavaScriptSourceRange; readonly identityReferenceLocation: JavaScriptSourceRange | null; + /** + * Range of the unwrapped returned expression. Null when the return has no + * value. This is the value's syntax, which may be wider than an identifier. + */ + readonly valueExpressionLocation: JavaScriptSourceRange | null; readonly value: JavaScriptSemanticValue; } @@ -128,6 +133,8 @@ export interface JavaScriptSemanticCallSite { export interface JavaScriptSemanticCallArgument { readonly index: number; readonly location: JavaScriptSourceRange; + /** Unwrapped value expression; null for a spread or non-expression slot. */ + readonly valueExpressionLocation: JavaScriptSourceRange | null; readonly spread: boolean; } diff --git a/src/domain/javascript/javascriptSemanticReturns.ts b/src/domain/javascript/javascriptSemanticReturns.ts index 61a90203a0..a34b7be780 100644 --- a/src/domain/javascript/javascriptSemanticReturns.ts +++ b/src/domain/javascript/javascriptSemanticReturns.ts @@ -54,6 +54,10 @@ export const collectSemanticReturns = ( const expressions = directReturnExpressions(node); const sites: JavaScriptSemanticReturnSite[] = []; for (const expression of expressions) { + const returned = + expression.node === null + ? null + : unwrapJavaScriptExpression(expression.node).node; sites.push({ returnSiteId: semanticReturnSiteId( callable.callableId, @@ -61,9 +65,10 @@ export const collectSemanticReturns = ( ), location: expression.location, identityReferenceLocation: - expression.node !== null && t.isIdentifier(expression.node) - ? range(expression.node) + returned !== null && t.isIdentifier(returned) + ? range(returned) : null, + valueExpressionLocation: returned === null ? null : range(returned), value: expression.node === null ? { status: "unknown", reason: "Return has no value." } diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index 30bb2c176b..f238ee9700 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -630,6 +630,7 @@ const callable = z.strictObject({ returnSiteId: z.string(), location: range, identityReferenceLocation: range.nullable(), + valueExpressionLocation: range.nullable(), value: semanticValue, }), ), diff --git a/tests/boundary/filesystem/javascriptApplicationInputSelection.test.ts b/tests/boundary/filesystem/javascriptApplicationInputSelection.test.ts index 390ea6664c..e0af6b86da 100644 --- a/tests/boundary/filesystem/javascriptApplicationInputSelection.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationInputSelection.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { chmod, mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it, onTestFinished } from "vitest"; @@ -23,6 +23,74 @@ const application = async () => { }; describe("JavaScript application input selection", () => { + it("preserves the selected path and host reason for an overlong name", async () => { + const { root } = await application(); + const longPath = join(root, "x".repeat(300)); + + expect(await analyzeFailure({ input_path: longPath })).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["input_path"], + reason: "invalid_value", + message: expect.stringContaining( + `Selected input path exceeds the host path or component length limit (ENAMETOOLONG): ${longPath}`, + ), + }, + ], + }, + }); + }); + + it.skipIf(process.platform === "win32")( + "preserves the selected path when symlink traversal loops or exceeds the host limit", + async () => { + const { root } = await application(); + const loopPath = join(root, "loop-a"); + await symlink("loop-b", loopPath); + await symlink("loop-a", join(root, "loop-b")); + await writeFile(join(root, "chain-end"), ""); + let target = "chain-end"; + for (let index = 59; index >= 0; index -= 1) { + const name = `chain-${String(index)}`; + await symlink(target, join(root, name)); + target = name; + } + const chainPath = join(root, "chain-0"); + + const loopError = await analyzeFailure({ input_path: loopPath }); + const chainError = await analyzeFailure({ input_path: chainPath }); + const eloopMessage = (path: string) => + `Selected input path could not be resolved because symbolic-link traversal looped or exceeded the host limit (ELOOP): ${path}`; + expect(loopError).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["input_path"], + reason: "invalid_value", + message: expect.stringContaining(eloopMessage(loopPath)), + }, + ], + }, + }); + expect(chainError).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["input_path"], + reason: "invalid_value", + message: expect.stringContaining(eloopMessage(chainPath)), + }, + ], + }, + }); + expect(loopError).not.toEqual(chainError); + }, + ); + it("reports a missing input path as invalid input, not an artifact read failure", async () => { const { root } = await application(); const missing = join(root, "missing"); From ef59174c1edd6ed9adb0722b86a5b9a57217e310 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:45:34 +0800 Subject: [PATCH 03/35] fix(errors): name fixed allowances in resource-constraint recovery (#5633) * fix(errors): name fixed allowances in resource-constraint recovery Fixed parser, count, report, input, and capture refusals inherited host-memory, wait-and-retry, or retained-Evidence advice. Producers now attach allowance-specific recovery, and deterministic capacity caps are not retryable. IL2CPP reader occupancy stays retryable capacity. Refs #5556, #5405, #5456, #5459, #5460, #5504, #5505, #5506, #5537, #5628, #5630, #5631, #5632 * fix(il2cpp): keep a closing metadata reader non-retryable close() stops admission before the active inspection finishes. A second inspect in that window was projected as retryable capacity. Closing and closed readers now report capability_unavailable. Occupied readers that remain open stay retryable. * fix(errors): align fixed-cap recovery with retained facts IL2CPP report refusals keep observed source facts instead of denying Evidence. Command-output caps use the capture allowance. Ghidra seed recovery names the limit that was exceeded. * fix(errors): call byte caps a byte allowance WASM representation limits and the IL2CPP Ghidra seed-file cap are byte budgets. Their recovery no longer says the exceeded allowance is a fixed count. * test(ida): keep unresolved procedure lookups non-retryable Main now treats an unchanged invalid request as non-retryable. The IDA lookup tests follow that projection. * fix(errors): qualify capture recovery for live output A fixed capture allowance still rejects the same output. Live captures such as adb logcat can change between requests, so recovery no longer says every unchanged request fails again. * fix(adb): keep a truncated live capture retryable Device output can shrink between calls, so an adb capture that exceeds the allowance stays retryable. Deterministic capture and version-output limits stay non-retryable. * fix(adb): keep truncated mutations non-retryable Only live observations opt into retrying a capture that exceeded the allowance. Push, install, uninstall, and other mutations stay non-retryable. * fix(errors): keep live capture recovery off deterministic limits Artifact-derived capture allowances again say the unchanged request fails. Only volatile adb observations use the live-output sentence. --------- Co-authored-by: Cursor Agent --- src/adb/AdbCommand.test.ts | 37 +++++++ src/adb/AdbCommand.ts | 17 +++- src/adb/AdbConfiguration.ts | 9 +- src/adb/AdbProvider.ts | 18 +++- src/apktool/ApktoolProvider.ts | 5 + .../WasmInterfaceComparisonService.test.ts | 73 +++++++++++++- .../wasm/WasmInterfaceComparisonService.ts | 5 + src/captures/TsharkCaptureProvider.ts | 13 ++- .../WebNetworkProjectionAccumulator.ts | 5 + src/domain/fixedAllowanceRemediation.test.ts | 67 +++++++++++++ src/domain/fixedAllowanceRemediation.ts | 49 ++++++++++ src/dotnet/ManagedSnapshot.ts | 2 + src/electron/ElectronFuseFailures.ts | 11 ++- src/electron/ElectronFuseProvider.test.ts | 11 +++ src/flutter/FlutterBuildProvider.ts | 23 ++++- src/go/GoBinaryProvider.ts | 14 ++- src/il2cpp/Il2cppGhidraSeeds.ts | 48 +++++++-- src/il2cpp/Il2cppMetadataProvider.ts | 44 ++++++++- src/native/AppleDispatchMetadata.ts | 5 + src/native/NativeMacOSProvider.ts | 9 +- src/native/pe/PeResourceParser.ts | 4 +- src/native/pe/PeResourcesProvider.test.ts | 29 ++++++ src/native/pe/PeResourcesProvider.ts | 38 ++++--- src/pcap/PcapCommand.ts | 9 +- src/pcap/PcapOutput.ts | 7 ++ src/pcap/PcapSnapshot.ts | 5 + src/process/ArtifactAnalysisResources.test.ts | 41 ++++++++ src/process/ArtifactAnalysisResources.ts | 9 +- src/rust/RustBinaryProvider.ts | 23 ++++- src/signals/SigrokRecordingProvider.ts | 10 +- src/sqlite/SqliteDatabaseWorkerExecution.ts | 6 +- src/wasm/WasmInterfaceBudget.test.ts | 36 +++++++ src/wasm/WasmInterfaceBudget.ts | 2 + .../boundary/captures/captureProvider.test.ts | 14 ++- .../filesystem/flutterBuildProvider.test.ts | 6 +- tests/boundary/go/goBinaryProvider.test.ts | 30 ++++++ .../boundary/il2cpp/il2cppGhidraSeeds.test.ts | 40 +++++++- .../il2cpp/il2cppMetadataProvider.test.ts | 98 ++++++++++++++++++- tests/boundary/rust/rustBinaryBudget.test.ts | 51 ++++++++++ .../signals/recordingProvider.test.ts | 16 +++ 40 files changed, 888 insertions(+), 51 deletions(-) create mode 100644 src/domain/fixedAllowanceRemediation.test.ts create mode 100644 src/domain/fixedAllowanceRemediation.ts create mode 100644 src/wasm/WasmInterfaceBudget.test.ts create mode 100644 tests/boundary/rust/rustBinaryBudget.test.ts diff --git a/src/adb/AdbCommand.test.ts b/src/adb/AdbCommand.test.ts index 52882c8112..33123950a9 100644 --- a/src/adb/AdbCommand.test.ts +++ b/src/adb/AdbCommand.test.ts @@ -1,4 +1,5 @@ import { expect, it, vi } from "vitest"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { ExecFileOutputCancelledError, execFileBytes, @@ -183,6 +184,42 @@ it("does not infer cancellation from a foreign ABORT_ERR code", async () => { }); }); +it.each([true, false])( + "projects a truncated adb capture as retryable only for volatile output (%s)", + async (volatileOutput) => { + vi.mocked(execFileBytes).mockRejectedValueOnce( + Object.assign(new Error("too much"), { + code: 1, + stdout: "", + stderr: "", + signal: null, + killed: false, + outputTruncated: true, + }), + ); + const result = await runAdbCapture( + volatileOutput ? "read_adb_logcat" : "install_adb_package", + { device: "device" }, + "/host/adb", + volatileOutput ? ["logcat", "-d"] : ["install", "app.apk"], + { timeoutMs: 100, maxOutputBytes: 1024, volatileOutput }, + undefined, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected a capture allowance refusal"); + const projected = projectAnalysisError(result.error); + expect(projected.retryable).toBe(volatileOutput); + expect(projected.remediation.action).toContain( + "The adb output capture allowance", + ); + expect(projected.remediation.action).toContain( + volatileOutput + ? "live output may change between requests" + : "Repeating the unchanged request fails the same way.", + ); + }, +); + it("keeps a physical failure even when the later abort reason is that same error", async () => { const controller = new AbortController(); const failure = new Error("physical command failure"); diff --git a/src/adb/AdbCommand.ts b/src/adb/AdbCommand.ts index c494fd71fd..b1859e385c 100644 --- a/src/adb/AdbCommand.ts +++ b/src/adb/AdbCommand.ts @@ -9,6 +9,7 @@ import type { AnalysisCapturedOutput, AnalysisError, } from "../domain/analysisErrorBase.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { err, ok, type Result } from "../domain/result.js"; import { execFileBytes, @@ -86,6 +87,11 @@ export const adbArguments = ( export interface AdbRunLimits { readonly timeoutMs: number; readonly maxOutputBytes: number; + /** + * Live device observations can shrink between calls. Mutation commands omit + * this so a truncated result stays non-retryable. + */ + readonly volatileOutput?: boolean; } /** Keep the selected device separate from the file addressed by the command. */ @@ -139,7 +145,14 @@ export const runAdbCapture = async ( "memory", `adb output exceeded the ${String(limits.maxOutputBytes)}-byte capture budget`, { max_output_bytes: limits.maxOutputBytes }, - { cause }, + { + cause, + ...fixedAllowanceOptions( + "The adb output capture allowance", + limits.volatileOutput === true ? "live" : "capture", + ), + ...(limits.volatileOutput === true ? { retryable: true } : {}), + }, ), ); const stderr = failure.stderr.trim(); @@ -268,7 +281,7 @@ export const probeAdbPathExistence = async ( { device: serial, path }, binary, adbArguments(serial, ["shell", "test", "-e", path]), - limits, + { ...limits, volatileOutput: true }, signal, ); if (probed.ok) { diff --git a/src/adb/AdbConfiguration.ts b/src/adb/AdbConfiguration.ts index bf5187b499..1de39f9c09 100644 --- a/src/adb/AdbConfiguration.ts +++ b/src/adb/AdbConfiguration.ts @@ -12,6 +12,7 @@ import { } from "../process/ExecFileOutput.js"; import type { ProviderAvailability } from "../application/AnalysisProvider.js"; import type { ProviderRejectionCode } from "../contracts/providerSelection.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { adbCommandCapturedOutput } from "./AdbCommand.js"; @@ -146,7 +147,13 @@ export const inspectAdbClient = async ( "memory", `adb version output exceeded the ${String(VERSION_MAX_OUTPUT_BYTES)}-byte capture budget`, { max_output_bytes: VERSION_MAX_OUTPUT_BYTES }, - options, + { + ...options, + ...fixedAllowanceOptions( + "The adb version-output capture allowance", + "capture", + ), + }, ); if (failure?.code === "ETIMEDOUT") throw new AnalysisTimeoutError( diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index e56c0448ab..677aedf379 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -1326,7 +1326,7 @@ export class AdbProvider { "serial" in request.input ? request.input.serial : undefined, ["shell", ...arguments_], ), - limits, + { ...limits, volatileOutput: true }, signal, ); switch (operation) { @@ -1346,6 +1346,7 @@ export class AdbProvider { { timeoutMs: DEVICES_TIMEOUT_MS, maxOutputBytes: DEVICES_MAX_OUTPUT_BYTES, + volatileOutput: true, }, signal, ); @@ -1382,6 +1383,7 @@ export class AdbProvider { { timeoutMs: SHELL_TIMEOUT_MS, maxOutputBytes: GETPROP_MAX_OUTPUT_BYTES, + volatileOutput: true, }, signal, ); @@ -1631,6 +1633,7 @@ export class AdbProvider { { timeoutMs: LISTING_TIMEOUT_MS, maxOutputBytes: LISTING_MAX_OUTPUT_BYTES, + volatileOutput: true, }, signal, ); @@ -1917,6 +1920,7 @@ export class AdbProvider { { timeoutMs: PM_PATH_TIMEOUT_MS, maxOutputBytes: PM_PATH_MAX_OUTPUT_BYTES, + volatileOutput: true, }, signal, ); @@ -2704,7 +2708,11 @@ export class AdbProvider { "android.intent.category.LAUNCHER", input.package, ]), - { timeoutMs: SHELL_TIMEOUT_MS, maxOutputBytes: PM_PATH_MAX_OUTPUT_BYTES }, + { + timeoutMs: SHELL_TIMEOUT_MS, + maxOutputBytes: PM_PATH_MAX_OUTPUT_BYTES, + volatileOutput: true, + }, signal, ); if (!resolved.ok) return resolved; @@ -2918,7 +2926,11 @@ export class AdbProvider { { device: input.serial }, selection.binary, adbArguments(input.serial, ["exec-out", "screencap", "-p"]), - { timeoutMs: SCREEN_TIMEOUT_MS, maxOutputBytes: SCREEN_MAX_OUTPUT_BYTES }, + { + timeoutMs: SCREEN_TIMEOUT_MS, + maxOutputBytes: SCREEN_MAX_OUTPUT_BYTES, + volatileOutput: true, + }, signal, ); if (!captured.ok) return captured; diff --git a/src/apktool/ApktoolProvider.ts b/src/apktool/ApktoolProvider.ts index 1bccd050e8..97a3cef52e 100644 --- a/src/apktool/ApktoolProvider.ts +++ b/src/apktool/ApktoolProvider.ts @@ -45,6 +45,7 @@ import { AnalysisUnsupportedTargetError, } from "../domain/analysisErrorCore.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { err, ok, type Result } from "../domain/result.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; @@ -194,6 +195,10 @@ const apktoolCommandFailure = ( { cause, ...(capturedOutput === undefined ? {} : { capturedOutput }), + ...fixedAllowanceOptions( + "The apktool output capture allowance", + "capture", + ), }, ); // A physical launcher/capture failure does not establish target refusal. diff --git a/src/application/wasm/WasmInterfaceComparisonService.test.ts b/src/application/wasm/WasmInterfaceComparisonService.test.ts index 9936c7530b..85fee9985f 100644 --- a/src/application/wasm/WasmInterfaceComparisonService.test.ts +++ b/src/application/wasm/WasmInterfaceComparisonService.test.ts @@ -1,8 +1,13 @@ import { expect, it } from "vitest"; import { createEvidence } from "../../domain/evidence.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { wasmInterfaceComparisonSchema } from "../../contracts/wasm/wasmComparisonContracts.js"; import { compareWasmInterfaces } from "./WasmInterfaceComparisonService.js"; -import type { WasmInterface } from "../../domain/wasm/wasmInterface.js"; +import { + WASM_INTERFACE_REPRESENTATION_BYTES, + wasmInterfaceEntryRepresentationBytes, + type WasmInterface, +} from "../../domain/wasm/wasmInterface.js"; import { wasmArtifactFixture } from "../../../tests/fixtures/wasm/artifact.js"; const facts = ( @@ -179,6 +184,72 @@ it("resolves exact retained references and permits cancellation between comparis }); }); +/** One inventory under the comparison cap; two of them exceed it together. */ +const admittedInventory = (label: string): WasmInterface => { + const base = facts(label).entries[0]!; + const minimum = Math.floor(WASM_INTERFACE_REPRESENTATION_BYTES / 2) + 4096; + let low = 0; + let high = 4_000_000; + let best = 0; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const name = `${label}-${"n".repeat(mid)}`; + const bytes = wasmInterfaceEntryRepresentationBytes({ + ...base, + name, + wat_form: `(export "${name}" (func 0))`, + }); + if (bytes >= minimum) { + best = mid; + high = mid - 1; + } else low = mid + 1; + } + const name = `${label}-${"n".repeat(best)}`; + const entry = { + ...base, + name, + wat_form: `(export "${name}" (func 0))`, + }; + const bytes = wasmInterfaceEntryRepresentationBytes(entry); + if ( + bytes >= WASM_INTERFACE_REPRESENTATION_BYTES || + bytes + 512 >= WASM_INTERFACE_REPRESENTATION_BYTES + ) + throw new Error("Sized inventory is not individually admitted"); + return { state: "complete", issues: [], entries: [entry] }; +}; + +it("refuses two admitted inventories that together exceed the comparison cap", async () => { + const left = admittedInventory("left"); + const right = admittedInventory("right"); + const empty: WasmInterface = { state: "complete", issues: [], entries: [] }; + const alone = await compareWasmInterfaces({ + left: evidence(left), + right: evidence(empty), + scope: "both", + }); + expect(alone.ok).toBe(true); + const combined = await compareWasmInterfaces({ + left: evidence(left), + right: evidence(right), + scope: "both", + }); + expect(combined.ok).toBe(false); + if (combined.ok) throw new Error("Expected comparison allowance refusal"); + const projected = projectAnalysisError(combined.error); + expect(projected.retryable).toBe(false); + expect(projected.code).toBe("resource_constraint"); + expect(combined.error).toMatchObject({ resource: "capacity" }); + expect(projected.remediation.action).toContain( + "The WASM interface comparison inventory", + ); + expect(projected.remediation.action).toContain("fixed byte allowance"); + expect(projected.remediation.action).not.toContain("fixed count"); + expect(projected.remediation.action).not.toMatch( + /Wait for an active analysis/u, + ); +}); + it.each([false, true])( "keeps a one-sided unsupported interface unknown with reverse=%s", async (reverse) => { diff --git a/src/application/wasm/WasmInterfaceComparisonService.ts b/src/application/wasm/WasmInterfaceComparisonService.ts index f920a5deaa..32e1c80865 100644 --- a/src/application/wasm/WasmInterfaceComparisonService.ts +++ b/src/application/wasm/WasmInterfaceComparisonService.ts @@ -22,6 +22,7 @@ import { AnalysisInputError, AnalysisResourceConstraintError, } from "../../domain/analysisErrorCore.js"; +import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../../domain/result.js"; @@ -133,6 +134,10 @@ export const compareWasmInterfaces = async ( "capacity", "WASM comparison exceeds typed interface representation capacity", { maximum_bytes: WASM_INTERFACE_REPRESENTATION_BYTES }, + fixedAllowanceOptions( + "The WASM interface comparison inventory", + "bytes", + ), ); if (++work % 256 === 0) { await setImmediate(); diff --git a/src/captures/TsharkCaptureProvider.ts b/src/captures/TsharkCaptureProvider.ts index 8acada8e5a..110f0f5dd2 100644 --- a/src/captures/TsharkCaptureProvider.ts +++ b/src/captures/TsharkCaptureProvider.ts @@ -45,6 +45,7 @@ import { AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; import { createEvidence } from "../domain/evidence.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { type InspectPacketCaptureInput, @@ -480,6 +481,10 @@ export class TsharkCaptureProvider implements PacketCapturePort { ...(observedOutput === undefined ? {} : { capturedOutput: observedOutput }), + ...fixedAllowanceOptions( + "The packet-field discovery record allowance", + "capture", + ), }, ); const reason = cause instanceof Error ? cause.message : String(cause); @@ -561,7 +566,13 @@ export class TsharkCaptureProvider implements PacketCapturePort { "file-size", reason, { input_bytes: LIMITS.input_bytes }, - { cause }, + { + cause, + ...fixedAllowanceOptions( + "the fixed packet-capture input allowance", + "input", + ), + }, ); if (cause instanceof ArtifactReaderFailure && cause.reason === "integrity") return new AnalysisArtifactChangedError(operation, path, reason, { diff --git a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts index e512ba4f69..2f9563f019 100644 --- a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts +++ b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { fixedAllowanceOptions } from "../fixedAllowanceRemediation.js"; import { AnalysisResourceConstraintError, AnalysisUnsupportedTargetError, @@ -58,6 +59,10 @@ export abstract class WebNetworkProjectionAccumulator { "memory", `${this.budgetSubject} retained state and temporary work exceed the projection representation budget.`, { output_bytes: this.maximumBytes }, + fixedAllowanceOptions( + "The analysis-view projection allowance", + "parser", + ), ), ); } diff --git a/src/domain/fixedAllowanceRemediation.test.ts b/src/domain/fixedAllowanceRemediation.test.ts new file mode 100644 index 0000000000..7691620689 --- /dev/null +++ b/src/domain/fixedAllowanceRemediation.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { AnalysisResourceConstraintError } from "./analysisErrorCore.js"; +import { projectAnalysisError } from "./analysisErrorProjection.js"; +import { + fixedAllowanceOptions, + fixedAllowanceRemediation, + type FixedAllowanceKind, +} from "./fixedAllowanceRemediation.js"; + +const forbidden = { + parser: [ + /available host memory\. Retry/u, + /worker memory limits and available/u, + ], + count: [ + /Wait for an active analysis/u, + /\bretry\b/iu, + /fixed byte allowance/u, + ], + bytes: [/Wait for an active analysis/u, /\bretry\b/iu, /fixed count/u], + report: [/get_evidence_bundle/u, /complete CLI JSON output/u], + input: [/evidence reply/u, /file-size allowance for the evidence/u], + capture: [/available host memory\. Retry/u, /live output may change/u], + live: [/available host memory\. Retry/u, /unchanged request/u], + work: [/worker CPU limits and observed signal/u], +} as const; + +describe("fixed allowance recovery", () => { + it.each([ + ["parser", "memory"], + ["count", "capacity"], + ["bytes", "capacity"], + ["report", "transport"], + ["input", "file-size"], + ["capture", "memory"], + ["live", "memory"], + ["work", "cpu"], + ] as const)( + "projects %s allowances without the generic %s fallback", + (kind: FixedAllowanceKind, resource) => { + const allowance = `The ${kind} allowance`; + const error = new AnalysisResourceConstraintError( + "inspect_fixture", + resource, + "allowance exceeded", + { maximum: 1 }, + fixedAllowanceOptions(allowance, kind), + ); + const projected = projectAnalysisError(error); + expect(projected.retryable).toBe(false); + expect(projected.remediation.action).toBe( + fixedAllowanceRemediation(allowance, kind), + ); + expect(projected.remediation.action).toContain(allowance); + for (const pattern of forbidden[kind]) + expect(projected.remediation.action).not.toMatch(pattern); + if (kind === "live") + expect(projected.remediation.action).toContain( + "live output may change between requests", + ); + if (kind === "capture") + expect(projected.remediation.action).toContain( + "Repeating the unchanged request fails the same way.", + ); + }, + ); +}); diff --git a/src/domain/fixedAllowanceRemediation.ts b/src/domain/fixedAllowanceRemediation.ts new file mode 100644 index 0000000000..a121a7f15d --- /dev/null +++ b/src/domain/fixedAllowanceRemediation.ts @@ -0,0 +1,49 @@ +/** + * Recovery for a deterministic allowance. Host memory, worker limits, waiting, + * and a different output channel do not raise it. + */ +export type FixedAllowanceKind = + | "parser" + | "count" + | "bytes" + | "report" + | "input" + | "capture" + | "live" + | "work"; + +const unchangedRequest = "Repeating the unchanged request fails the same way."; + +/** Name the fixed allowance and the recovery that cannot admit the same request. */ +export const fixedAllowanceRemediation = ( + allowance: string, + kind: FixedAllowanceKind, +): string => { + switch (kind) { + case "parser": + return `${allowance} is a fixed allowance. Worker memory limits, inherited limits, and additional host memory do not raise it. Select another complete input within the allowance when that preserves the needed observation. ${unchangedRequest}`; + case "count": + return `${allowance} is a fixed count. Waiting for an active analysis to finish does not raise it. ${unchangedRequest}`; + case "bytes": + return `${allowance} is a fixed byte allowance. Waiting for an active analysis to finish does not raise it. ${unchangedRequest}`; + case "report": + return `${allowance} is a fixed report allowance. This refusal does not leave a completed report or Evidence record to retrieve, and complete CLI JSON does not raise it. ${unchangedRequest}`; + case "input": + return `Select a suitable complete artifact within ${allowance}. Increasing an evidence-reply file-size allowance does not admit this input. Do not truncate required bytes. ${unchangedRequest}`; + case "capture": + return `${allowance} is a fixed capture allowance. Worker memory limits, inherited limits, and additional host memory do not raise it. ${unchangedRequest}`; + case "live": + return `${allowance} is a fixed capture allowance. Worker memory limits, inherited limits, and additional host memory do not raise it. Capturing the same output exceeds the allowance again; live output may change between requests.`; + case "work": + return `${allowance} is a fixed work allowance. Worker CPU limits and additional CPU time do not raise it. Select another complete artifact within the allowance. ${unchangedRequest}`; + } +}; + +/** Non-retryable options for a fixed allowance. Occupied capacity stays retryable. */ +export const fixedAllowanceOptions = ( + allowance: string, + kind: FixedAllowanceKind, +): { readonly retryable: false; readonly remediationAction: string } => ({ + retryable: false, + remediationAction: fixedAllowanceRemediation(allowance, kind), +}); diff --git a/src/dotnet/ManagedSnapshot.ts b/src/dotnet/ManagedSnapshot.ts index 45d1111cf3..4edfd89f84 100644 --- a/src/dotnet/ManagedSnapshot.ts +++ b/src/dotnet/ManagedSnapshot.ts @@ -17,6 +17,7 @@ import { AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { RegularFileChangedError } from "../filesystem/RegularFile.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; @@ -42,6 +43,7 @@ const managedSnapshotCapacityError = ( "memory", "Managed static inspection requires an eagerly retained PE snapshot within its input byte budget.", { max_snapshot_bytes: MAX_MANAGED_SNAPSHOT_BYTES }, + fixedAllowanceOptions("The managed PE snapshot allowance", "parser"), ); export const readManagedSnapshot = async ( diff --git a/src/electron/ElectronFuseFailures.ts b/src/electron/ElectronFuseFailures.ts index e7658c3a03..40d88a8dcd 100644 --- a/src/electron/ElectronFuseFailures.ts +++ b/src/electron/ElectronFuseFailures.ts @@ -17,6 +17,7 @@ import { AnalysisResourceConstraintError, AnalysisUnsupportedTargetError, } from "../domain/analysisErrorCore.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ZodError } from "zod"; import { @@ -93,7 +94,15 @@ export const electronFuseFailure = ( cause.boundary === "file-size" ? { maximum_input_bytes: 1024 * 1024 * 1024 } : { maximum_sentinels_per_slice: 32 }, - { cause }, + { + cause, + ...fixedAllowanceOptions( + cause.boundary === "file-size" + ? "the 1 GiB Electron fuse input allowance" + : "The Electron fuse sentinel inventory", + cause.boundary === "file-size" ? "input" : "count", + ), + }, ); if (cause instanceof ElectronFuseFormatFailure) return cause.kind === "unsupported" diff --git a/src/electron/ElectronFuseProvider.test.ts b/src/electron/ElectronFuseProvider.test.ts index dd6227caaf..57c41ac773 100644 --- a/src/electron/ElectronFuseProvider.test.ts +++ b/src/electron/ElectronFuseProvider.test.ts @@ -575,8 +575,15 @@ it("enforces the 32 sentinel boundary with typed and public limit details", asyn expect(result.error._tag).toBe("AnalysisResourceConstraintError"); expect(projectAnalysisError(result.error)).toMatchObject({ code: "resource_constraint", + retryable: false, details: { reported_limits: { maximum_sentinels_per_slice: 32 } }, + remediation: { + action: expect.stringContaining("sentinel inventory"), + }, }); + expect( + projectAnalysisError(result.error).remediation.action, + ).not.toMatch(/Wait for an active analysis/u); } } } @@ -593,7 +600,11 @@ it("rejects a sparse artifact over 1 GiB before scanning", async () => { expect(result.error._tag).toBe("AnalysisResourceConstraintError"); expect(projectAnalysisError(result.error)).toMatchObject({ code: "resource_constraint", + retryable: false, details: { reported_limits: { maximum_input_bytes: 1024 * 1024 * 1024 } }, + remediation: { + action: expect.stringContaining("does not admit this input"), + }, }); } }); diff --git a/src/flutter/FlutterBuildProvider.ts b/src/flutter/FlutterBuildProvider.ts index ca37a58bc7..e67367e965 100644 --- a/src/flutter/FlutterBuildProvider.ts +++ b/src/flutter/FlutterBuildProvider.ts @@ -29,6 +29,7 @@ import { } from "../domain/analysisErrorCore.js"; import { AnalysisProtocolError } from "../domain/analysisErrorCore.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -74,6 +75,8 @@ const APK_MAX_BYTES = 2 * 1024 * 1024 * 1024; // engine labels this keeps expanded result metadata below 512 KiB for ordinary // ABI names, independently of the archive's compressed-byte budget. const MAX_FLUTTER_ABIS = 32; +const flutterBudget = (allowance: string, kind: "parser" | "input") => + fixedAllowanceOptions(allowance, kind); const MAX_POOL_URIS = 500; const MAX_POOL_TOKENS = 2_000; /** A complete AOT image carries exactly one VM and one isolate section. */ @@ -190,7 +193,10 @@ const mapInputFailure = ( "memory", cause.message, { max_metadata_read_bytes: APK_METADATA_READ_MAX_BYTES }, - { cause }, + { + cause, + ...flutterBudget("The Flutter APK metadata read allowance", "parser"), + }, ); return new AnalysisInputError(operation, { cause }, [ { @@ -296,6 +302,7 @@ const collectFlutterLibraries = async ( max_flutter_abis: MAX_FLUTTER_ABIS, max_library_path_bytes: MAX_FLUTTER_LIBRARY_PATH_BYTES, }, + flutterBudget("The Flutter ABI inventory allowance", "parser"), ); libraries.push({ entry, @@ -331,6 +338,7 @@ const checkApkSize = (operation: string, bytes: number): void => { "file-size", `The selected APK exceeds the ${String(APK_MAX_BYTES)}-byte snapshot budget`, { max_apk_bytes: APK_MAX_BYTES, observed_bytes: bytes }, + flutterBudget("the Flutter APK snapshot allowance", "input"), ); }; @@ -659,6 +667,7 @@ export class FlutterBuildProvider { "file-size", `${entry.path} exceeds the library read budget`, { max_library_bytes: LIBRARY_MAX_BYTES, entry_path: entry.path }, + flutterBudget("the Flutter library read allowance", "input"), ), ); } @@ -679,6 +688,7 @@ export class FlutterBuildProvider { max_total_bytes: TOTAL_EXTRACTION_MAX_BYTES, declared_total: declaredTotal, }, + flutterBudget("the Flutter cumulative extraction allowance", "input"), ), ); @@ -730,6 +740,7 @@ export class FlutterBuildProvider { max_library_bytes: LIBRARY_MAX_BYTES, entry_path: entry.path, }, + flutterBudget("the Flutter library read allowance", "input"), ), ); hash.update(view); @@ -747,6 +758,10 @@ export class FlutterBuildProvider { "file-size", `cumulative library extraction across ${String(flutterEntries.length)} entries exceeds the ${String(TOTAL_EXTRACTION_MAX_BYTES)}-byte budget`, { max_total_bytes: TOTAL_EXTRACTION_MAX_BYTES }, + flutterBudget( + "the Flutter cumulative extraction allowance", + "input", + ), ), ); } @@ -765,6 +780,10 @@ export class FlutterBuildProvider { max_snapshot_hash_candidates: MAX_SNAPSHOT_HASH_CANDIDATES, entry_path: entry.path, }, + flutterBudget( + "The Flutter snapshot-hash candidate allowance", + "parser", + ), ), ); facts.set(abi, { @@ -965,6 +984,7 @@ export class FlutterBuildProvider { "file-size", `lib/${selected.abi}/libapp.so declares ${String(declared)} bytes, exceeding the ${String(LIBRARY_MAX_BYTES)}-byte read budget`, { max_library_bytes: LIBRARY_MAX_BYTES }, + flutterBudget("the Flutter library read allowance", "input"), ), ); const imagePath = join(snapshot.root, "libapp.so"); @@ -994,6 +1014,7 @@ export class FlutterBuildProvider { "file-size", `lib/${selected.abi}/libapp.so exceeds the ${String(LIBRARY_MAX_BYTES)}-byte read budget while streaming`, { max_library_bytes: LIBRARY_MAX_BYTES }, + flutterBudget("the Flutter library read allowance", "input"), ), ); imageHash.update(view); diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 250e9a288d..7bbeb5c1d5 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -27,6 +27,7 @@ import { type InspectGoBinaryInput, } from "../domain/go/goBinary.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -62,7 +63,17 @@ const failureForPrimary = (cause: unknown, path: string): AnalysisError => { cause.boundary === "search" ? "cpu" : "memory", `Selected artifact ${path}: ${cause.message}`, { boundary: cause.boundary, maximum_bytes: cause.maximum_bytes }, - { cause }, + { + cause, + ...fixedAllowanceOptions( + cause.boundary === "search" + ? "The Go build-info header-search allowance" + : cause.boundary === "build-info" + ? "The Go build-info string allowance" + : "The Go structural decoding allowance", + cause.boundary === "search" ? "work" : "parser", + ), + }, ); if (cause instanceof GoBinaryFormatFailure) return cause.kind === "unsupported" @@ -152,6 +163,7 @@ const outputLimit = (path: string) => "transport", `Complete Go metadata for ${path} exceeds the inline output budget; no partial success is returned.`, { maximum_output_bytes: OUTPUT_BYTES }, + fixedAllowanceOptions("The Go complete-report allowance", "report"), ); /** Fixed reader boundaries plus unknowns left by the observed metadata. */ diff --git a/src/il2cpp/Il2cppGhidraSeeds.ts b/src/il2cpp/Il2cppGhidraSeeds.ts index 2210068398..a19a987c76 100644 --- a/src/il2cpp/Il2cppGhidraSeeds.ts +++ b/src/il2cpp/Il2cppGhidraSeeds.ts @@ -7,12 +7,24 @@ import { AnalysisInputError, AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; const OPERATION = "inspect_il2cpp_metadata"; -const capacity = (message: string) => - new AnalysisResourceConstraintError(OPERATION, "capacity", message, { - maximum_seed_bytes: MAX_GHIDRA_SEED_BYTES, - maximum_seed_name_characters: MAX_GHIDRA_SEED_NAME_LENGTH, - }); +const capacity = ( + message: string, + allowance: string, + kind: "bytes" | "count", +) => + new AnalysisResourceConstraintError( + OPERATION, + "capacity", + message, + { + maximum_seed_bytes: MAX_GHIDRA_SEED_BYTES, + maximum_seed_entries: MAX_GHIDRA_SEED_ENTRIES, + maximum_seed_name_characters: MAX_GHIDRA_SEED_NAME_LENGTH, + }, + fixedAllowanceOptions(allowance, kind), + ); import type { Il2cppMetadata } from "../domain/il2cpp/il2cppMetadata.js"; /** A linked metadata report reduced to Ghidra function seeds. */ @@ -72,11 +84,19 @@ const typeName = (report: Il2cppMetadata, index: number): string => { ]); length += type.name.length + 1; if (length > MAX_GHIDRA_SEED_NAME_LENGTH) - throw capacity("Qualified seed type name exceeds the Ghidra name limit."); + throw capacity( + "Qualified seed type name exceeds the Ghidra name limit.", + "The IL2CPP Ghidra seed-name allowance", + "count", + ); parts.unshift(type.name); namespace = type.namespace; if (length + namespace.length > MAX_GHIDRA_SEED_NAME_LENGTH) - throw capacity("Qualified seed type name exceeds the Ghidra name limit."); + throw capacity( + "Qualified seed type name exceeds the Ghidra name limit.", + "The IL2CPP Ghidra seed-name allowance", + "count", + ); at = type.declaring_type; } return namespace === "" ? parts.join(".") : `${namespace}.${parts.join(".")}`; @@ -112,7 +132,11 @@ export const il2cppGhidraSeeds = ( const typeNames = new Map(); const reserve = (length: number) => { if (length > MAX_GHIDRA_SEED_BYTES - bytes) - throw capacity("Seed file exceeds Ghidra byte admission."); + throw capacity( + "Seed file exceeds Ghidra byte admission.", + "The IL2CPP Ghidra seed-file allowance", + "bytes", + ); bytes += length; }; const emit = (text: string) => { @@ -144,6 +168,8 @@ export const il2cppGhidraSeeds = ( if (ownerName.length + 2 + method.name.length > MAX_GHIDRA_SEED_NAME_LENGTH) throw capacity( "Qualified method seed name exceeds Ghidra character admission.", + "The IL2CPP Ghidra seed-name allowance", + "count", ); const name = `${ownerName}$$${method.name}`.replace(/\s/gu, "_"); if (name.includes("\0")) @@ -169,7 +195,11 @@ export const il2cppGhidraSeeds = ( } seeded.set(address, { kept: name, also: [] }); if (seeded.size > MAX_GHIDRA_SEED_ENTRIES) - throw capacity("Seed function count exceeds Ghidra entry admission."); + throw capacity( + "Seed function count exceeds Ghidra entry admission.", + "The IL2CPP Ghidra seed-entry allowance", + "count", + ); emit(`0x${address.toString(16)}\tfunction\t${name}\n`); }); const foldedGroups = [...seeded] diff --git a/src/il2cpp/Il2cppMetadataProvider.ts b/src/il2cpp/Il2cppMetadataProvider.ts index d3561d94d5..b215ec461a 100644 --- a/src/il2cpp/Il2cppMetadataProvider.ts +++ b/src/il2cpp/Il2cppMetadataProvider.ts @@ -27,6 +27,7 @@ import { AnalysisAccessDeniedError, AnalysisArtifactChangedError, AnalysisCancelledError, + AnalysisCapabilityUnavailableError, AnalysisInputError, AnalysisOutputError, AnalysisResourceConstraintError, @@ -37,6 +38,7 @@ import { type Il2cppMetadata, type Il2cppMetadataProviderInput, } from "../domain/il2cpp/il2cppMetadata.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -76,7 +78,13 @@ const failureForPrimary = ( cause.reason === "native-type-text-capacity" ? "memory" : "capacity", cause.message, {}, - { cause }, + { + cause, + ...fixedAllowanceOptions( + "The IL2CPP native-link allowance", + cause.reason === "native-type-text-capacity" ? "parser" : "count", + ), + }, ); if ( cause instanceof Il2cppBinaryFormatFailure || @@ -112,7 +120,10 @@ const failureForPrimary = ( "file-size", cause.message, { maximum_input_bytes: maximum }, - { cause }, + { + cause, + ...fixedAllowanceOptions("the fixed IL2CPP input allowance", "input"), + }, ); if (cause.reason === "path") return new AnalysisInputError(OPERATION, { cause }, [ @@ -220,12 +231,20 @@ const binaryReport = ( method_addresses: link.method_addresses, }); +/** + * Source facts stay on the refusal. The shared report sentence would deny an + * Evidence record that this reader attaches and MCP retains. + */ +const completeReportRecovery = + "The IL2CPP complete-report allowance is a fixed report allowance. No complete metadata inventory is retained. Source facts already observed stay with this refusal, and complete CLI JSON does not raise the allowance. Repeating the unchanged request fails the same way."; + const reportLimit = (path: string, maximumBytes: number) => new AnalysisResourceConstraintError( OPERATION, "memory", `Complete IL2CPP metadata for ${path} exceeds the retained report budget; no complete inventory is retained.`, { maximum_report_bytes: maximumBytes }, + { retryable: false, remediationAction: completeReportRecovery }, ); const sourceArtifact = (path: string, snapshot: StableArtifactSnapshot) => ({ @@ -316,6 +335,8 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { readonly identity = IL2CPP_METADATA_PROVIDER_IDENTITY; readonly #resources = new ArtifactResourceScope(); #inspecting = false; + /** `close()` stops admission permanently, including while an inspection is active. */ + #admissionStopped = false; /** `outputBytes` exists so tests can reach the guard with small files. */ constructor( @@ -324,6 +345,7 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { ) {} async close(): Promise { + this.#admissionStopped = true; try { await this.#resources.close(); } catch (cause: unknown) { @@ -352,13 +374,29 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { return err( new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), ); + if (this.#admissionStopped) + return err( + new AnalysisCapabilityUnavailableError( + IL2CPP_METADATA_PROVIDER_IDENTITY.id, + OPERATION, + "IL2CPP metadata reader is closed", + { + remediationAction: + "This reader has stopped admitting inspections. Create a new IL2CPP metadata reader to inspect again. Repeating the request on this reader fails the same way.", + }, + ), + ); if (this.#inspecting) return err( new AnalysisResourceConstraintError( OPERATION, - "memory", + "capacity", "Only one IL2CPP metadata inspection is admitted per reader.", { maximum_concurrent_inspections: 1 }, + { + remediationAction: + "Retry after the active IL2CPP metadata inspection on this reader completes.", + }, ), ); this.#inspecting = true; diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index 3a455c839f..f5343bbfe2 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -24,6 +24,7 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { @@ -217,6 +218,10 @@ export const inspectAppleDispatchMetadata = async ( "memory", "Apple static metadata input exceeds its 64 MiB snapshot budget", { input_bytes: maximumBytes }, + fixedAllowanceOptions( + "The Apple static metadata snapshot allowance", + "parser", + ), ); const cancelled = () => new AnalysisCancelledError("inspect_native_dispatch_metadata", { diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 145c3d43a5..7541e84398 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -55,6 +55,7 @@ import { AnalysisResourceConstraintError, AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { @@ -1216,7 +1217,13 @@ const translateCommandFailure = ( "memory", `Native command output exceeded ${String(NATIVE_COMMAND_OUTPUT_BUDGET_BYTES)} bytes; complete output is required for analysis.`, { max_output_bytes: NATIVE_COMMAND_OUTPUT_BUDGET_BYTES }, - errorOptions, + { + ...errorOptions, + ...fixedAllowanceOptions( + "The native command output capture allowance", + "capture", + ), + }, ), ); return bindCleanup( diff --git a/src/native/pe/PeResourceParser.ts b/src/native/pe/PeResourceParser.ts index 1f7829268a..1f266defb3 100644 --- a/src/native/pe/PeResourceParser.ts +++ b/src/native/pe/PeResourceParser.ts @@ -19,7 +19,9 @@ type Range = { offset: number; bytes: number }; type OccupiedRange = Range & { kind: "directory" | "name" | "data-entry" | "payload"; }; -const METADATA_BYTES = 8 * 1024 * 1024; +/** Complete PE resource result retained before a refusal. */ +export const PE_RESOURCE_METADATA_BYTES = 8 * 1024 * 1024; +const METADATA_BYTES = PE_RESOURCE_METADATA_BYTES; const limit = (message: string, boundary: ArtifactReaderBoundary): never => { throw new ArtifactReaderFailure("limit", message, { boundary }); }; diff --git a/src/native/pe/PeResourcesProvider.test.ts b/src/native/pe/PeResourcesProvider.test.ts index f54100c22e..62d7502d68 100644 --- a/src/native/pe/PeResourcesProvider.test.ts +++ b/src/native/pe/PeResourcesProvider.test.ts @@ -172,3 +172,32 @@ describe("PE resource provider budget reporting", () => { }); }); }); + +it("names the fixed PE metadata allowance instead of host memory", async () => { + const root = await createTestTempDirectory("rea-pe-metadata-budget-"); + const path = join(root, "resource.exe"); + const name = "n".repeat(200); + const resources = Array.from({ length: 5500 }, (_, index) => ({ + type: 10, + name: `${name}${String(index).padStart(4, "0")}`, + language: 1033, + payload: Buffer.alloc(0), + })); + await writeFile( + path, + peResourceFixture(false, resources, 12 * 1024 * 1024).bytes, + ); + const result = await inspect(path); + if (result.ok) throw new Error("Expected a metadata refusal"); + const error = result.error as AnalysisResourceConstraintError; + expect(error.resource).toBe("memory"); + expect(error.reportedLimits).toMatchObject({ + maximum_metadata_bytes: 8 * 1024 * 1024, + }); + const projected = projectAnalysisError(error); + expect(projected.retryable).toBe(false); + expect(projected.remediation.action).toContain( + "PE resource metadata allowance", + ); + expect(projected.remediation.action).not.toMatch(/available host memory/u); +}); diff --git a/src/native/pe/PeResourcesProvider.ts b/src/native/pe/PeResourcesProvider.ts index bd62e9e972..702cda02f5 100644 --- a/src/native/pe/PeResourcesProvider.ts +++ b/src/native/pe/PeResourcesProvider.ts @@ -1,3 +1,4 @@ +import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; import { @@ -25,7 +26,10 @@ import { import { err, ok } from "../../domain/result.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { PE_RESOURCES_PROVIDER } from "../../application/InvestigationProviders.js"; -import { parsePeResources } from "./PeResourceParser.js"; +import { + parsePeResources, + PE_RESOURCE_METADATA_BYTES, +} from "./PeResourceParser.js"; /** Portable parser over one bounded, identity-checked regular file. */ export class PeResourcesProvider implements PeResourcesPort { @@ -118,7 +122,8 @@ const peResourcesFailurePrimary = ( if (cause instanceof ArtifactReaderFailure) { // An input byte budget is a file-size bound, not a memory constraint, and // an entry budget is a capacity bound. Report what was actually exceeded. - if (cause.reason === "limit") + if (cause.reason === "limit") { + const metadata = cause.boundary === "memory"; return new AnalysisResourceConstraintError( operation, cause.boundary ?? "memory", @@ -126,22 +131,31 @@ const peResourcesFailurePrimary = ( { max_file_bytes: input.max_file_bytes, max_entries: input.max_entries, + ...(metadata + ? { maximum_metadata_bytes: PE_RESOURCE_METADATA_BYTES } + : {}), }, { cause, - // Both bounds are caller-selected and deterministic for the same - // input, so waiting for an active analysis cannot help. - ...(cause.boundary === undefined || cause.boundary === "memory" + // Caller-selected budgets and the fixed metadata allowance are + // deterministic for the same input, so waiting cannot help. + ...(cause.boundary === undefined ? {} - : { - retryable: false, - remediationAction: inputBudgetRemediation( - cause.boundary, - input, - ), - }), + : metadata + ? fixedAllowanceOptions( + "The PE resource metadata allowance", + "parser", + ) + : { + retryable: false as const, + remediationAction: inputBudgetRemediation( + cause.boundary, + input, + ), + }), }, ); + } if (cause.reason === "path" || cause.reason === "format") return new AnalysisInputError(operation, { cause }, [ { diff --git a/src/pcap/PcapCommand.ts b/src/pcap/PcapCommand.ts index be4ea4564f..8e16648e5b 100644 --- a/src/pcap/PcapCommand.ts +++ b/src/pcap/PcapCommand.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, @@ -223,7 +224,13 @@ export const runPcapCommand = async (options: { "memory", "Tool output exceeded its complete-output budget", { max_output_bytes: limits.maxOutputBytes }, - details, + { + ...details, + ...fixedAllowanceOptions( + "The packet-tool output capture allowance", + "capture", + ), + }, ); if ( cause instanceof WindowsBatchCommandError || diff --git a/src/pcap/PcapOutput.ts b/src/pcap/PcapOutput.ts index 3944d3871e..7226e46370 100644 --- a/src/pcap/PcapOutput.ts +++ b/src/pcap/PcapOutput.ts @@ -5,6 +5,7 @@ import { AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; import { parseRecordedTransportUrl } from "../domain/recordedTransportUrl.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; const OPERATION = "inspect_pcap_capture"; @@ -161,6 +162,7 @@ export const parseTsharkJsonOutput = async ( "memory", "tshark JSON exceeds the bounded decoding budget", { max_json_bytes: TSHARK_JSON_MAX_BYTES }, + fixedAllowanceOptions("The tshark JSON decoding allowance", "capture"), ); // The selected-field protocol has five container levels. Reject deeper // graphs before materializing them; quoted brackets are string data. @@ -192,6 +194,7 @@ export const parseTsharkJsonOutput = async ( "memory", "tshark JSON expansion exceeds the decoding budget", { max_decoded_bytes: MAX_DECODED_BYTES }, + fixedAllowanceOptions("The tshark JSON expansion allowance", "capture"), ); } let packets: unknown; @@ -220,6 +223,10 @@ export const parseTsharkJsonOutput = async ( "memory", "Packet inventory exceeds the retained-fact budget", { max_retained_bytes: MAX_RETAINED_BYTES }, + fixedAllowanceOptions( + "The packet-inventory retained-fact allowance", + "capture", + ), ); }; const add = (set: Set, value: string): void => { diff --git a/src/pcap/PcapSnapshot.ts b/src/pcap/PcapSnapshot.ts index fb1620d86e..b4e3d7919d 100644 --- a/src/pcap/PcapSnapshot.ts +++ b/src/pcap/PcapSnapshot.ts @@ -11,6 +11,7 @@ import { AnalysisInputError, AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { @@ -156,6 +157,10 @@ export class PcapSnapshot { "file-size", "Capture exceeds the owned-snapshot disk budget", { max_capture_bytes: MAX_CAPTURE_BYTES }, + fixedAllowanceOptions( + "the owned capture snapshot allowance", + "input", + ), ); const destination = this.own( new OwnedFileHandle(await open(this.path, "wx", 0o600)), diff --git a/src/process/ArtifactAnalysisResources.test.ts b/src/process/ArtifactAnalysisResources.test.ts index 1536c0a4ae..2e59422afb 100644 --- a/src/process/ArtifactAnalysisResources.test.ts +++ b/src/process/ArtifactAnalysisResources.test.ts @@ -491,6 +491,47 @@ it.each([false, true])( }, ); +it("treats a fixed command-output cap as a non-retryable capture allowance", async () => { + const resources = new ArtifactAnalysisResources( + "fixture", + "inspect_packet_fields", + readStableArtifact, + readLinuxResourceLimits, + { message: "busy", details: {} }, + ); + const failure = new OwnedCommandFailure( + "output-limit", + "command output exceeded the retained limit", + null, + ); + try { + const error = resources.commandFailure( + failure, + { command_timeout_ms: 100, output_bytes: 1024 }, + undefined, + ); + expect(error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "memory", + reportedLimits: { output_bytes: 1024 }, + retryable: false, + }); + const projected = projectAnalysisError(error); + expect(projected.retryable).toBe(false); + expect(projected.remediation.action).toContain( + "The retained command-output allowance", + ); + expect(projected.remediation.action).not.toMatch( + /Review the reported worker memory limits/u, + ); + expect(projected.remediation.action).not.toMatch( + /available host memory\. Retry/u, + ); + } finally { + await resources.close(); + } +}); + it("retains a physical command failure when the owner also observed late cancellation", async () => { const resources = new ArtifactAnalysisResources( "fixture", diff --git a/src/process/ArtifactAnalysisResources.ts b/src/process/ArtifactAnalysisResources.ts index 35f2d4703d..29168213a1 100644 --- a/src/process/ArtifactAnalysisResources.ts +++ b/src/process/ArtifactAnalysisResources.ts @@ -22,6 +22,7 @@ import { } from "../domain/analysisErrorCore.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -362,7 +363,13 @@ export class ArtifactAnalysisResources { "memory", cause.message, { output_bytes: limits.output_bytes }, - detail, + { + ...detail, + ...fixedAllowanceOptions( + "The retained command-output allowance", + "capture", + ), + }, ); if ( cause.reason === "process" && diff --git a/src/rust/RustBinaryProvider.ts b/src/rust/RustBinaryProvider.ts index 7e1b826677..1f88804b5b 100644 --- a/src/rust/RustBinaryProvider.ts +++ b/src/rust/RustBinaryProvider.ts @@ -26,6 +26,7 @@ import { type InspectRustBinaryInput, type RustBinary, } from "../domain/rust/rustBinary.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; import { readRustBinaryImage } from "./RustBinaryImage.js"; @@ -120,6 +121,7 @@ const outputLimit = (path: string) => "transport", `Complete Rust metadata for ${path} exceeds the inline output budget; no partial success is returned.`, { maximum_output_bytes: RUST_OUTPUT_BYTES }, + fixedAllowanceOptions("The Rust complete-report allowance", "report"), ); const limitationsFor = ( @@ -186,7 +188,21 @@ const failureForPrimary = (cause: unknown, path: string): AnalysisError => { cause.constraint.resource, `Selected artifact ${path}: ${cause.message}`, cause.constraint.reported_limits, - { cause }, + { + cause, + ...fixedAllowanceOptions( + cause.constraint.resource === "capacity" + ? "The rustc commit-path inventory" + : cause.constraint.resource === "transport" + ? "The Rust complete-report allowance" + : "This Rust inspection allowance", + cause.constraint.resource === "capacity" + ? "count" + : cause.constraint.resource === "transport" + ? "report" + : "parser", + ), + }, ); if (cause instanceof RustAuditableFormatFailure) return cause.kind === "unsupported" @@ -274,7 +290,10 @@ const artifactReaderFailure = ( "file-size", cause.message, { maximum_input_bytes: RUST_BINARY_INPUT_BYTES }, - { cause }, + { + cause, + ...fixedAllowanceOptions("the fixed Rust input allowance", "input"), + }, ); if (cause.reason === "path") return new AnalysisInputError(OPERATION, { cause }, [ diff --git a/src/signals/SigrokRecordingProvider.ts b/src/signals/SigrokRecordingProvider.ts index e623e02985..6b8918d576 100644 --- a/src/signals/SigrokRecordingProvider.ts +++ b/src/signals/SigrokRecordingProvider.ts @@ -43,6 +43,7 @@ import { AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; import { createEvidence } from "../domain/evidence.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { windowAnnotationText } from "../domain/signals/annotationTextWindow.js"; import { @@ -439,6 +440,10 @@ export class SigrokRecordingProvider implements SignalRecordingPort { ...(observedOutput === undefined ? {} : { capturedOutput: observedOutput }), + ...fixedAllowanceOptions( + "The sigrok decoder-discovery line allowance", + "capture", + ), }, ); const reason = cause instanceof Error ? cause.message : String(cause); @@ -520,7 +525,10 @@ export class SigrokRecordingProvider implements SignalRecordingPort { "file-size", reason, { input_bytes: LIMITS.input_bytes }, - { cause }, + { + cause, + ...fixedAllowanceOptions("the fixed sigrok input allowance", "input"), + }, ); if (cause instanceof ArtifactReaderFailure && cause.reason === "integrity") return new AnalysisArtifactChangedError(operation, path, reason, { diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index 4d06f9c07a..d3fea33924 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -8,6 +8,7 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { z } from "zod"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, @@ -97,7 +98,10 @@ const rejectedReply = ( sqlite_value_or_record_bytes: SQLITE_DATABASE_LIMITS.outputBytes, worker_reply_bytes: SQLITE_DATABASE_LIMITS.outputBytes, }, - { capturedOutput }, + { + capturedOutput, + ...fixedAllowanceOptions("The SQLite worker reply allowance", "report"), + }, ); return new AnalysisInputError(operation, { capturedOutput }, [ { diff --git a/src/wasm/WasmInterfaceBudget.test.ts b/src/wasm/WasmInterfaceBudget.test.ts new file mode 100644 index 0000000000..9f4edd5fde --- /dev/null +++ b/src/wasm/WasmInterfaceBudget.test.ts @@ -0,0 +1,36 @@ +import { expect, it } from "vitest"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { WASM_INTERFACE_REPRESENTATION_BYTES } from "../domain/wasm/wasmInterface.js"; +import { WasmInterfaceBudget } from "./WasmInterfaceBudget.js"; + +it("refuses typed WASM interface growth without asking the caller to wait", () => { + const budget = new WasmInterfaceBudget(); + expect(() => + budget.reserve(WASM_INTERFACE_REPRESENTATION_BYTES + 1), + ).toThrowError(); + try { + budget.reserve(WASM_INTERFACE_REPRESENTATION_BYTES + 1); + throw new Error("Expected a representation refusal"); + } catch (error) { + const projected = projectAnalysisError( + error as Parameters[0], + ); + expect(projected).toMatchObject({ + retryable: false, + details: { + resource: "capacity", + reported_limits: { + maximum_bytes: WASM_INTERFACE_REPRESENTATION_BYTES, + }, + }, + }); + expect(projected.remediation.action).toContain( + "typed WASM interface inventory", + ); + expect(projected.remediation.action).toContain("fixed byte allowance"); + expect(projected.remediation.action).not.toContain("fixed count"); + expect(projected.remediation.action).not.toMatch( + /Wait for an active analysis/u, + ); + } +}); diff --git a/src/wasm/WasmInterfaceBudget.ts b/src/wasm/WasmInterfaceBudget.ts index 0d928b7ffb..dc910ea2b3 100644 --- a/src/wasm/WasmInterfaceBudget.ts +++ b/src/wasm/WasmInterfaceBudget.ts @@ -1,4 +1,5 @@ import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { WASM_INTERFACE_REPRESENTATION_BYTES } from "../domain/wasm/wasmInterface.js"; /** Eager typed observations share the existing WABT output capacity, not an agent token estimate. */ @@ -11,6 +12,7 @@ export class WasmInterfaceBudget { "capacity", "Typed WASM interface decoding exceeds the WABT output representation budget", { maximum_bytes: WASM_INTERFACE_REPRESENTATION_BYTES }, + fixedAllowanceOptions("The typed WASM interface inventory", "bytes"), ); this.#remaining -= bytes; } diff --git a/tests/boundary/captures/captureProvider.test.ts b/tests/boundary/captures/captureProvider.test.ts index a7ec58b29f..9265e8120c 100644 --- a/tests/boundary/captures/captureProvider.test.ts +++ b/tests/boundary/captures/captureProvider.test.ts @@ -1,3 +1,4 @@ +import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; import { packetFieldsCatalogSchema } from "../../../src/domain/captures/packetFields.js"; import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; import { it, expect, onTestFinished } from "vitest"; @@ -613,11 +614,22 @@ it.each([ }); if (mode === "invalid-json") expect(result.error.capturedOutput?.stdout).toBe("not json\n"); - if (mode === "overflow") + if (mode === "overflow") { + expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(result.error instanceof AnalysisResourceConstraintError)) + throw new Error("Expected command-output allowance refusal"); expect(result.error).toMatchObject({ resource: "memory", reportedLimits: { output_bytes: 32 * 1024 * 1024 }, + retryable: false, }); + expect(result.error.remediationAction).toContain( + "The retained command-output allowance", + ); + expect(result.error.remediationAction).not.toMatch( + /Review the reported worker memory limits/u, + ); + } }); it("retains completed discovery on late cancellation and cleanup failure", async () => { diff --git a/tests/boundary/filesystem/flutterBuildProvider.test.ts b/tests/boundary/filesystem/flutterBuildProvider.test.ts index 289f159ba3..9bcc4ac1d7 100644 --- a/tests/boundary/filesystem/flutterBuildProvider.test.ts +++ b/tests/boundary/filesystem/flutterBuildProvider.test.ts @@ -928,7 +928,11 @@ it.each(["identify_flutter_build", "inspect_dart_aot"] as const)( const result = await provider.execute({ operation, input: { path } }); if (result.ok) throw new Error("expected a size refusal"); expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError); - expect(projectAnalysisError(result.error).code).toBe("resource_constraint"); + const projected = projectAnalysisError(result.error); + expect(projected.code).toBe("resource_constraint"); + expect(projected.retryable).toBe(false); + expect(projected.remediation.action).toContain("APK snapshot allowance"); + expect(projected.remediation.action).not.toMatch(/evidence reply/u); expect(result.error).toMatchObject({ operation, reportedLimits: { diff --git a/tests/boundary/go/goBinaryProvider.test.ts b/tests/boundary/go/goBinaryProvider.test.ts index 7c9c9a5475..e59484d03e 100644 --- a/tests/boundary/go/goBinaryProvider.test.ts +++ b/tests/boundary/go/goBinaryProvider.test.ts @@ -7,9 +7,11 @@ import { AnalysisResourceConstraintError, AnalysisUnsupportedTargetError, } from "../../../src/domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { createGoBinaryDiagnosticFixtures, createGoBinaryFixture, + createGoOverlappingElfFixture, } from "../../fixtures/go/image.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -67,6 +69,34 @@ it("preserves the actual build-info byte guard as a resource failure", async () boundary: "build-info", maximum_bytes: 1024 * 1024, }); + const projected = projectAnalysisError(result.error); + expect(projected.retryable).toBe(false); + expect(projected.remediation.action).toContain( + "Go build-info string allowance", + ); + expect(projected.remediation.action).not.toMatch(/available host memory/u); +}); + +it("keeps the header-search budget distinct from a worker CPU limit", async () => { + const root = await createTestTempDirectory("rea-go-search-"); + const path = join(root, "overlapping.elf"); + await writeFile(path, createGoOverlappingElfFixture()); + const result = await new GoBinaryProvider().inspect({ path }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Overlapping header search must be refused"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + retryable: false, + details: { + resource: "cpu", + reported_limits: { + boundary: "search", + maximum_bytes: 256 * 1024 * 1024, + }, + }, + }); + expect(projected.remediation.action).toContain("header-search allowance"); + expect(projected.remediation.action).not.toMatch(/observed signal/u); }); it.skipIf(process.platform === "win32")( diff --git a/tests/boundary/il2cpp/il2cppGhidraSeeds.test.ts b/tests/boundary/il2cpp/il2cppGhidraSeeds.test.ts index d063b37fbf..45b33ed455 100644 --- a/tests/boundary/il2cpp/il2cppGhidraSeeds.test.ts +++ b/tests/boundary/il2cpp/il2cppGhidraSeeds.test.ts @@ -1,4 +1,7 @@ -import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../../../src/domain/analysisErrorCore.js"; import { expect, it } from "vitest"; import { parseGhidraSeeds } from "../../../src/ghidra/GhidraAnalysisSeeds.js"; import { @@ -175,7 +178,23 @@ it("bounds folded-name retention against the actual Ghidra file admission", () = })), method_addresses: Array.from({ length: count }, () => address("0x100")), }; - expect(() => il2cppGhidraSeeds(repeated, 0n)).toThrow(/byte admission/); + let caught: unknown; + try { + il2cppGhidraSeeds(repeated, 0n); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(caught instanceof AnalysisResourceConstraintError)) + throw new Error("Expected seed-file allowance refusal"); + expect(caught.message).toMatch(/byte admission/); + expect(caught.remediationAction).toContain( + "The IL2CPP Ghidra seed-file allowance", + ); + expect(caught.remediationAction).toContain("fixed byte allowance"); + expect(caught.remediationAction).not.toContain("fixed count"); + expect(caught.remediationAction).not.toContain("seed-name allowance"); + expect(caught.retryable).toBe(false); }); it("refuses cyclic type ancestry before emitting a seed", () => { const cyclic = { @@ -200,5 +219,20 @@ it("refuses names the real Ghidra seed parser cannot admit", () => { ...report, methods: [{ name: "M".repeat(2001), declaring_type: 0 }], }; - expect(() => il2cppGhidraSeeds(oversized, 0n)).toThrow(/character admission/); + let caught: unknown; + try { + il2cppGhidraSeeds(oversized, 0n); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(caught instanceof AnalysisResourceConstraintError)) + throw new Error("Expected seed-name allowance refusal"); + expect(caught.message).toMatch(/character admission/); + expect(caught.remediationAction).toContain( + "The IL2CPP Ghidra seed-name allowance", + ); + expect(caught.remediationAction).toContain("fixed count"); + expect(caught.remediationAction).not.toContain("fixed byte allowance"); + expect(caught.retryable).toBe(false); }); diff --git a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts index 0c28878fc5..c944b04355 100644 --- a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts @@ -5,8 +5,10 @@ import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; import { AnalysisCancelledError, + AnalysisCapabilityUnavailableError, AnalysisResourceConstraintError, } from "../../../src/domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { Il2cppMetadataProvider } from "../../../src/il2cpp/Il2cppMetadataProvider.js"; import { createElfFixture, @@ -188,6 +190,41 @@ it("admits a report whose escaped names fit the budget exactly", async () => { } }); +it("keeps observed source facts with a complete-report refusal", async () => { + const root = await createTestTempDirectory("rea-il2cpp-report-evidence-"); + const path = join(root, "global-metadata.dat"); + const bytes = createIl2cppMetadataFixture().bytes; + await writeFile(path, bytes); + const result = await inspectMetadata(path, 256); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected report allowance refusal"); + expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError); + expect(result.error.partialObservation).toMatchObject({ + operation: "observe_il2cpp_metadata_source", + normalized_result: { + artifact: { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + inventory_status: "not-returned-as-complete", + }, + }); + const projected = projectAnalysisError(result.error); + expect(projected.retryable).toBe(false); + expect(projected.code).toBe("resource_constraint"); + expect(projected.remediation.action).toContain( + "No complete metadata inventory is retained", + ); + expect(projected.remediation.action).toContain( + "Source facts already observed stay with this refusal", + ); + expect(projected.remediation.action).not.toMatch( + /does not leave a completed report or Evidence record/u, + ); + expect(projected.remediation.action).not.toMatch(/worker memory/u); +}); + it("refuses from row counts before decoding any row", async () => { const root = await createTestTempDirectory("rea-il2cpp-provider-counts-"); const path = join(root, "global-metadata.dat"); @@ -288,8 +325,17 @@ it("bounds concurrent snapshots and retains active decoding until close complete try { const refused = await provider.inspect({ path }); expect(refused.ok).toBe(false); - if (!refused.ok) - expect(refused.error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!refused.ok) { + expect(refused.error).toBeInstanceOf(AnalysisCapabilityUnavailableError); + expect(projectAnalysisError(refused.error)).toMatchObject({ + code: "capability_unavailable", + retryable: false, + details: { reason: "IL2CPP metadata reader is closed" }, + }); + expect( + projectAnalysisError(refused.error).remediation.action, + ).not.toMatch(/retry/i); + } await setImmediate(); expect(closed).toBe(false); } finally { @@ -298,7 +344,53 @@ it("bounds concurrent snapshots and retains active decoding until close complete } expect((await active).ok).toBe(true); expect(closed).toBe(true); - expect((await provider.inspect({ path })).ok).toBe(false); + const afterClose = await provider.inspect({ path }); + expect(afterClose.ok).toBe(false); + if (!afterClose.ok) { + expect(afterClose.error).toBeInstanceOf(AnalysisCapabilityUnavailableError); + expect(projectAnalysisError(afterClose.error).retryable).toBe(false); + } +}); + +it("keeps occupied capacity retryable while the reader remains open", async () => { + const root = await createTestTempDirectory("rea-il2cpp-provider-occupied-"); + const path = join(root, "global-metadata.dat"); + await writeFile(path, createIl2cppMetadataFixture().bytes); + let markEntered!: () => void; + const entered = new Promise((resolve) => { + markEntered = resolve; + }); + let resume!: () => void; + const release = new Promise((resolve) => { + resume = resolve; + }); + const provider = new Il2cppMetadataProvider(undefined, () => { + markEntered(); + return release; + }); + const active = provider.inspect({ path }); + await entered; + try { + const refused = await provider.inspect({ path }); + expect(refused.ok).toBe(false); + if (!refused.ok) { + expect(refused.error).toBeInstanceOf(AnalysisResourceConstraintError); + expect(projectAnalysisError(refused.error)).toMatchObject({ + retryable: true, + details: { + resource: "capacity", + reported_limits: { maximum_concurrent_inspections: 1 }, + }, + remediation: { + action: expect.stringContaining("active IL2CPP metadata inspection"), + }, + }); + } + } finally { + resume(); + await active; + await provider.close(); + } }); it.each([false, true])( diff --git a/tests/boundary/rust/rustBinaryBudget.test.ts b/tests/boundary/rust/rustBinaryBudget.test.ts new file mode 100644 index 0000000000..8b498c62ab --- /dev/null +++ b/tests/boundary/rust/rustBinaryBudget.test.ts @@ -0,0 +1,51 @@ +import { truncate, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { RustBinaryProvider } from "../../../src/rust/RustBinaryProvider.js"; +import { RUST_BINARY_INPUT_BYTES } from "../../../src/rust/RustAuditableReader.js"; +import { createRustHashOverflowImage } from "../../fixtures/rust/image.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("refuses extra rustc paths without asking the caller to wait", async () => { + const root = await createTestTempDirectory("rea-rust-hash-budget-"); + const path = join(root, "many-hashes.elf"); + await writeFile(path, createRustHashOverflowImage()); + const result = await new RustBinaryProvider().inspect({ path }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected a hash-capacity refusal"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + retryable: false, + details: { + resource: "capacity", + reported_limits: { maximum_rustc_hashes: 64 }, + }, + }); + expect(projected.remediation.action).toContain("rustc commit-path inventory"); + expect(projected.remediation.action).not.toMatch( + /Wait for an active analysis/u, + ); +}); + +it("refuses an oversized Rust input without evidence-reply advice", async () => { + const root = await createTestTempDirectory("rea-rust-input-budget-"); + const path = join(root, "large.elf"); + await writeFile(path, Buffer.from([0x7f, 0x45, 0x4c, 0x46])); + await truncate(path, RUST_BINARY_INPUT_BYTES + 1); + const result = await new RustBinaryProvider().inspect({ path }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected an input-size refusal"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + retryable: false, + details: { + resource: "file-size", + reported_limits: { maximum_input_bytes: RUST_BINARY_INPUT_BYTES }, + }, + }); + expect(projected.remediation.action).toContain("does not admit this input"); + expect(projected.remediation.action).not.toMatch( + /file-size allowance for the evidence reply/u, + ); +}); diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 8b6c608e99..0a75aa5671 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -1,3 +1,4 @@ +import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; import { signalDecodersSchema } from "../../../src/domain/signals/signalDecoders.js"; import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; import { it, expect, onTestFinished } from "vitest"; @@ -568,6 +569,21 @@ it.each([ stdout: "sigrok-cli 0.7.1\n", stderr: "version warning\n", }); + if (mode === "overflow") { + expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(result.error instanceof AnalysisResourceConstraintError)) + throw new Error("Expected command-output allowance refusal"); + expect(result.error).toMatchObject({ + resource: "memory", + retryable: false, + }); + expect(result.error.remediationAction).toContain( + "The retained command-output allowance", + ); + expect(result.error.remediationAction).not.toMatch( + /Review the reported worker memory limits/u, + ); + } }); it("retains completed discovery when cleanup fails, then retries before accepting work", async () => { From 40428da1dd6308f6f407547fd23b0a912232895a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:46:14 +0800 Subject: [PATCH 04/35] perf(test): reuse one stdio MCP server per JavaScript application file (#5658) * perf(test): reuse one stdio MCP server per JavaScript application file Repeated analyzeJavaScriptDirectory MCP cases started a full server for every case. Share one file-scoped server and compile cache so later cases skip that startup while CLI cases still launch a fresh process. Co-authored-by: morluto * fix(test): close in-flight shared MCP startup with the file File teardown waits for a stdio session that is still starting, skips a server once close has begun, and still closes one registered during that wait. The later-case reuse test can run without the earlier case. Co-authored-by: morluto * fix(test): keep a failed MCP handshake when cleanup fails The shared session reports the startup error and carries the cleanup failure as its cause. A cleanup that failed stays owned and is retried on close without running a cleanup that already succeeded. Co-authored-by: morluto * fix(test): fail shared MCP cleanup when the rejection has no reason A cleanup rejection of undefined still fails the drain and stays owned. Close caches completion only after every retained cleanup succeeds, so a later close can release it. Co-authored-by: morluto * fix(test): keep one MCP server while failed cleanup is retained A later call retries retained cleanup before opening another server. The new server starts only after that cleanup succeeds. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- docs/testing.md | 11 + .../cli/sharedJavaScriptMcpSession.test.ts | 264 ++++++++++++++++++ tests/support/cli/cliFixture.ts | 183 +++++++++++- tests/support/cli/javascriptApplication.ts | 22 +- 4 files changed, 473 insertions(+), 7 deletions(-) create mode 100644 tests/boundary/cli/sharedJavaScriptMcpSession.test.ts diff --git a/docs/testing.md b/docs/testing.md index 1ab5184aad..7dc7705ca1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -146,6 +146,17 @@ the fixture cache. These subprocesses are outside aggregate Vitest coverage; subprocess coverage attachment remains disabled. Cold-start import checks launch independently without this fixture cache. +`analyzeJavaScriptDirectory` stdio MCP cases in one `cliTest` file share one +server and that compile cache. The first case pays startup; later cases reuse +the process. File teardown waits for startup still in flight, closes a server +registered during that wait, and does not open a server after close. A failed +handshake stays the caller error when its cleanup also fails, including a +rejection without a reason. That cleanup stays owned until a later close or +call releases it. Another server does not open while the cleanup remains, and +a cleanup that already succeeded is not run again. Pass +`mcpCall` when a case needs its own server, environment, or lifecycle. CLI +cases still start a fresh process. + The Apple artifact verifier step shares a job-temporary bytecode cache among its fresh CLI/MCP children via `REA_ARTIFACT_NODE_COMPILE_CACHE`. The artifact helper maps this setting to child `NODE_COMPILE_CACHE`; npm, Turbo, and nested diff --git a/tests/boundary/cli/sharedJavaScriptMcpSession.test.ts b/tests/boundary/cli/sharedJavaScriptMcpSession.test.ts new file mode 100644 index 0000000000..ce281a7431 --- /dev/null +++ b/tests/boundary/cli/sharedJavaScriptMcpSession.test.ts @@ -0,0 +1,264 @@ +import { expect, test } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + cliTest, + createSharedMcpSession, + type SharedMcpCall, + type TestCli, +} from "../../support/cli/cliFixture.js"; +import { + analyzeJavaScriptDirectory, + resolveJavaScriptApplicationMcpCall, +} from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/workspace/writeFixtureFiles.js"; + +const unusedCli = { + run: async () => { + throw new Error("CLI was not requested"); + }, +} as TestCli; + +test("prefers an explicit JavaScript MCP caller over the file session", async () => { + const explicit = (async () => ({ + isError: false, + })) as unknown as SharedMcpCall; + let sharedCalls = 0; + const resolved = await resolveJavaScriptApplicationMcpCall( + { + ...unusedCli, + sharedMcpCall: async () => { + sharedCalls += 1; + return explicit; + }, + }, + explicit, + ); + expect(resolved).toBe(explicit); + expect(sharedCalls).toBe(0); +}); + +test("uses the file session when the case does not own a server", async () => { + const shared = (async () => ({ isError: false })) as unknown as SharedMcpCall; + const resolved = await resolveJavaScriptApplicationMcpCall( + { + ...unusedCli, + sharedMcpCall: async () => shared, + }, + undefined, + ); + expect(resolved).toBe(shared); +}); + +const writeApplication = async (prefix: string): Promise => { + const root = await createTestTempDirectory(prefix); + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "index.html": "page", + }); + return root; +}; + +let fileSession: SharedMcpCall | undefined; + +/** Record the first selected case and compare later cases when they also run. */ +const useFileSession = async (cli: TestCli): Promise => { + const open = cli.sharedMcpCall; + if (open === undefined) + throw new Error("cliTest did not provide a shared MCP session"); + const current = await open(); + if (fileSession === undefined) fileSession = current; + else expect(current).toBe(fileSession); + return current; +}; + +const held = (): { gate: Promise; release: () => void } => { + let release: () => void = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + return { gate, release }; +}; + +const fakeCall = (async () => ({ + isError: false, +})) as unknown as SharedMcpCall; + +test("does not open a shared MCP server after file cleanup starts", async () => { + const startup = held(); + let opened = false; + const session = createSharedMcpSession(async (_registerCleanup, isClosed) => { + await startup.gate; + if (isClosed()) throw new Error("Shared MCP session is closed"); + opened = true; + return fakeCall; + }); + const opening = session.session.call(); + const closing = session.close(); + startup.release(); + await closing; + await expect(opening).rejects.toThrow("Shared MCP session is closed"); + expect(opened).toBe(false); + await expect(session.session.call()).rejects.toThrow( + "Shared MCP session is closed", + ); +}); + +test("keeps a failed MCP handshake when its cleanup also fails", async () => { + const startup = new Error("handshake failed"); + const cleanup = new Error("teardown failed"); + let released = 0; + let retained = 0; + const session = createSharedMcpSession(async (registerCleanup) => { + registerCleanup(async () => { + retained += 1; + if (retained === 1) throw cleanup; + }); + registerCleanup(async () => { + released += 1; + }); + throw startup; + }); + const caught = await session.session.call().then( + () => { + throw new Error("startup succeeded"); + }, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(Error); + if (!(caught instanceof Error)) throw new Error("expected startup failure"); + expect(caught.message).toBe(startup.message); + expect(caught.cause).toBeInstanceOf(AggregateError); + const cause = caught.cause as AggregateError; + expect(cause.cause).toBe(startup); + expect([...cause.errors]).toEqual([startup, cleanup]); + expect(released).toBe(1); + expect(retained).toBe(1); + await session.close(); + expect(released).toBe(1); + expect(retained).toBe(2); +}); + +test("retries a reasonless MCP cleanup rejection on a later close", async () => { + const startup = new Error("handshake failed"); + let attempts = 0; + let fail = true; + const session = createSharedMcpSession(async (registerCleanup) => { + registerCleanup(() => { + attempts += 1; + return fail ? Promise.reject() : Promise.resolve(); + }); + throw startup; + }); + const caught = await session.session.call().then( + () => { + throw new Error("startup succeeded"); + }, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(Error); + if (!(caught instanceof Error)) throw new Error("expected startup failure"); + expect(caught.message).toBe(startup.message); + expect(caught.cause).toBeInstanceOf(AggregateError); + const cause = caught.cause as AggregateError; + expect(cause.cause).toBe(startup); + expect([...cause.errors]).toEqual([startup, undefined]); + await expect(session.close()).rejects.toBeUndefined(); + expect(attempts).toBe(2); + fail = false; + await session.close(); + expect(attempts).toBe(3); +}); + +test("does not open another MCP server while failed cleanup is retained", async () => { + const startup = new Error("handshake failed"); + const cleanup = new Error("teardown failed"); + let openings = 0; + let attempts = 0; + let failCleanup = true; + const session = createSharedMcpSession(async (registerCleanup) => { + openings += 1; + registerCleanup(async () => { + attempts += 1; + if (failCleanup) throw cleanup; + }); + throw startup; + }); + const first = await session.session.call().then( + () => { + throw new Error("startup succeeded"); + }, + (error: unknown) => error, + ); + expect(first).toBeInstanceOf(Error); + if (!(first instanceof Error)) throw new Error("expected startup failure"); + expect(first.message).toBe(startup.message); + expect(openings).toBe(1); + expect(attempts).toBe(1); + + const blocked = await session.session.call().then( + () => { + throw new Error("startup succeeded"); + }, + (error: unknown) => error, + ); + expect(blocked).toBe(cleanup); + expect(openings).toBe(1); + expect(attempts).toBe(2); + + failCleanup = false; + const reopened = await session.session.call().then( + () => { + throw new Error("startup succeeded"); + }, + (error: unknown) => error, + ); + expect(reopened).toBe(startup); + expect(openings).toBe(2); + expect(attempts).toBe(4); +}); + +test("closes a shared MCP server registered after file cleanup starts", async () => { + const startup = held(); + let closedServer = false; + const session = createSharedMcpSession(async (registerCleanup) => { + await startup.gate; + registerCleanup(async () => { + closedServer = true; + }); + return fakeCall; + }); + const opening = session.session.call(); + const closing = session.close(); + startup.release(); + await closing; + await expect(opening).rejects.toThrow("Shared MCP session is closed"); + expect(closedServer).toBe(true); +}); + +cliTest( + "opens one stdio MCP session for JavaScript application cases", + async ({ cli }) => { + const first = await useFileSession(cli); + expect(await useFileSession(cli)).toBe(first); + const root = await writeApplication("rea-shared-mcp-"); + const analyzed = await analyzeJavaScriptDirectory("stdio MCP", cli, root); + expect( + analyzed.graph.nodes.some(({ kind }) => kind === "electron-renderer"), + ).toBe(true); + }, +); + +cliTest( + "reuses that stdio MCP session for a later case in the file", + async ({ cli }) => { + await useFileSession(cli); + const root = await writeApplication("rea-shared-mcp-later-"); + const analyzed = await analyzeJavaScriptDirectory("stdio MCP", cli, root); + expect( + analyzed.graph.nodes.some(({ kind }) => kind === "electron-renderer"), + ).toBe(true); + }, +); diff --git a/tests/support/cli/cliFixture.ts b/tests/support/cli/cliFixture.ts index e41f6ea951..3ee43177a5 100644 --- a/tests/support/cli/cliFixture.ts +++ b/tests/support/cli/cliFixture.ts @@ -25,11 +25,174 @@ export interface CliResult extends ProcessResult { readonly json: unknown | undefined; } +/** One compiled stdio MCP tool call. */ +export type SharedMcpCall = Awaited< + ReturnType< + typeof import("../../fixtures/localToolsMcp.js").connectLocalToolsMcp + > +>["call"]; + /** Compiled REA CLI driver owned by one test. */ export interface TestCli { run(invocation: CliInvocation): Promise; + /** + * File-scoped stdio MCP caller. Present on `cliTest` drivers and absent for + * drivers created with `createTestCli` alone. + */ + sharedMcpCall?(): Promise; } +/** One file-scoped stdio MCP session. */ +export interface SharedMcpSession { + call(): Promise; +} + +const sharedMcpSessionClosed = (): Error => + new Error("Shared MCP session is closed"); + +/** Keep the handshake as the caller error and retain the cleanup failure beside it. */ +const startupFailureWithCleanup = ( + startup: unknown, + cleanup: unknown, +): Error => { + const primary = + startup instanceof Error ? startup : new Error(String(startup)); + return new Error(primary.message, { + cause: new AggregateError([primary, cleanup], primary.message, { + cause: primary, + }), + }); +}; + +/** + * Start at most one server for the file. Close waits for startup still in + * flight. A connector that sees `isClosed` can skip the server; a server + * registered after close is still cleaned up. A failed cleanup stays owned, + * and the next call does not open another server until that cleanup succeeds. + */ +export const createSharedMcpSession = ( + connect: ( + registerCleanup: (cleanup: () => Promise) => void, + isClosed: () => boolean, + ) => Promise, +): { session: SharedMcpSession; close(): Promise } => { + let pending: Promise | undefined; + let closed = false; + let closing: Promise | undefined; + let closeGeneration: symbol | undefined; + let cleanupQueue: Promise = Promise.resolve(); + const cleanups: Array<() => Promise> = []; + const drain = (): Promise => { + const run = cleanupQueue + .then( + () => undefined, + () => undefined, + ) + .then(async () => { + const failures: unknown[] = []; + const retained: Array<() => Promise> = []; + while (cleanups.length > 0) { + const cleanup = cleanups.pop(); + if (cleanup === undefined) break; + try { + await cleanup(); + } catch (error) { + failures.push(error); + retained.push(cleanup); + } + } + for (const cleanup of retained.reverse()) cleanups.push(cleanup); + // A reasonless rejection is `undefined` and still owns the cleanup. + if (failures.length > 1) { + throw new AggregateError( + failures, + "Shared MCP session cleanup failed", + ); + } + if (failures.length === 1) throw failures[0]; + }); + cleanupQueue = run; + return run; + }; + const registerCleanup = (cleanup: () => Promise): void => { + cleanups.push(cleanup); + }; + return { + session: { + call: () => { + if (closed) return Promise.reject(sharedMcpSessionClosed()); + pending ??= (async () => { + try { + if (cleanups.length > 0) await drain(); + } catch (cleanupError) { + // Close owns cleanup once shutdown has started, so a failed + // handshake cannot clear the startup close is awaiting. + if (!closed) pending = undefined; + throw cleanupError; + } + if (closed) throw sharedMcpSessionClosed(); + try { + const opened = await connect(registerCleanup, () => closed); + if (closed) throw sharedMcpSessionClosed(); + return opened; + } catch (error) { + if (!closed) { + pending = undefined; + try { + await drain(); + } catch (cleanupError) { + throw startupFailureWithCleanup(error, cleanupError); + } + } + throw error; + } + })(); + return pending; + }, + }, + close: () => { + closed = true; + if (closing !== undefined) return closing; + const starting = pending; + const generation = Symbol("shared-mcp-close"); + closeGeneration = generation; + const attempt = (async () => { + try { + if (starting !== undefined) { + await starting.catch(() => undefined); + } + await drain(); + } catch (error) { + // A failed close stays retryable. Completion is cached only after + // every retained cleanup has succeeded. + if (closeGeneration === generation) { + closing = undefined; + closeGeneration = undefined; + } + throw error; + } + })(); + closing = attempt; + return attempt; + }, + }; +}; + +const createFileMcpSession = ( + compileCache: string, +): { session: SharedMcpSession; close(): Promise } => + createSharedMcpSession(async (registerCleanup, isClosed) => { + const { connectLocalToolsMcp } = await import( + "../../fixtures/localToolsMcp.js" + ); + if (isClosed()) throw sharedMcpSessionClosed(); + const mcp = await connectLocalToolsMcp( + { NODE_COMPILE_CACHE: compileCache }, + registerCleanup, + ); + return mcp.call; + }); + const parseJson = (stdout: string): unknown | undefined => { const document = stdout.trim(); if (document.length === 0) return undefined; @@ -92,6 +255,7 @@ export const createTestCli = ( /** Vitest API extended with an owned process registry and compiled CLI driver. */ export const cliTest = processTest.extend<{ $test: { cli: TestCli }; + $file: { mcpSession: SharedMcpSession }; $worker: { cliCompileCache: string }; }>({ cliCompileCache: [ @@ -106,7 +270,22 @@ export const cliTest = processTest.extend<{ }, { scope: "worker" }, ], - cli: async ({ processes, cliCompileCache }, use) => { - await use(createTestCli(processes, cliCompileCache)); + mcpSession: [ + async ({ cliCompileCache }, use) => { + const opened = createFileMcpSession(cliCompileCache); + try { + await use(opened.session); + } finally { + await opened.close(); + } + }, + { scope: "file" }, + ], + cli: async ({ processes, cliCompileCache, mcpSession }, use) => { + const cli = createTestCli(processes, cliCompileCache); + await use({ + ...cli, + sharedMcpCall: () => mcpSession.call(), + }); }, }); diff --git a/tests/support/cli/javascriptApplication.ts b/tests/support/cli/javascriptApplication.ts index 59bb317f2d..6a04b9592c 100644 --- a/tests/support/cli/javascriptApplication.ts +++ b/tests/support/cli/javascriptApplication.ts @@ -6,7 +6,7 @@ import { z } from "zod"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; -import type { TestCli } from "./cliFixture.js"; +import type { SharedMcpCall, TestCli } from "./cliFixture.js"; const execute = promisify(execFile); @@ -21,6 +21,20 @@ export const runNodeJsonControl = async ( return JSON.parse(stdout) as unknown; }; +/** + * Use an explicit caller when the test owns the server. Otherwise reuse the + * file-scoped session from `cliTest`, and start a fresh server only for + * drivers created outside that fixture. + */ +export const resolveJavaScriptApplicationMcpCall = async ( + cli: TestCli, + mcpCall: SharedMcpCall | undefined, +): Promise => { + if (mcpCall !== undefined) return mcpCall; + if (cli.sharedMcpCall !== undefined) return cli.sharedMcpCall(); + return (await connectLocalToolsMcp()).call; +}; + /** Analyze a directory; a supplied MCP caller reuses its test-owned connection. */ export const analyzeJavaScriptDirectory = async ( surface: "CLI" | "stdio MCP", @@ -31,9 +45,7 @@ export const analyzeJavaScriptDirectory = async ( mcpCall, }: { readonly sourcePaths?: readonly string[]; - readonly mcpCall?: - | Awaited>["call"] - | undefined; + readonly mcpCall?: SharedMcpCall | undefined; } = {}, ) => { let document: unknown; @@ -51,7 +63,7 @@ export const analyzeJavaScriptDirectory = async ( expect(response.exitCode, response.stderr).toBe(0); document = response.json; } else { - const call = mcpCall ?? (await connectLocalToolsMcp()).call; + const call = await resolveJavaScriptApplicationMcpCall(cli, mcpCall); const response = await call("analyze_javascript_application", { input_path: root, format: "directory", From 9d8e7803326a382cbbb3ffdb900f5e93e5442d68 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:48:14 +0800 Subject: [PATCH 05/35] ci: keep npm download caches from Turbo snapshot growth (#5655) * ci: keep npm download caches from Turbo snapshot growth Per-commit static and portable Turbo snapshots had filled the 10 GB Actions cache, which is what makes warm npm ci cheap. Trusted static runs now keep two generations and one download-cache writer per host. Co-authored-by: morluto * fix(ci): declare cache retention types for strict checks Typecheck imports cache-retention.mjs and needs a sibling declaration. The generation count stays module-private so knip does not report it. Co-authored-by: morluto * fix(ci): keep compatible caches and qualification read-only Npm dedupe deletes a record only when another has the same key, version, and ref. Cache deletion moves to a follow-up workflow so reusable qualification can keep its read-only token. Co-authored-by: morluto * fix(ci): retain caches after failed default-branch runs Static and build save Turbo snapshots before later lanes finish. Retention now follows a failed, timed-out, or cancelled trusted run, not only a green one. Co-authored-by: morluto * test(cli): pass application trace input as a file The CLI already reads workflow JSON from a path. Putting the full application Evidence document on argv exceeds the runner argument limit. Co-authored-by: morluto * fix(ci): retain deep and Knip snapshots without deleting a repeated id Fast and deep static checks publish rea-static-deep-v1 and knip-meta-v1 entries per commit. Retention prunes those families with the same two-generation rule. A cache id listed twice is one record; inconsistent repeats abort before any delete. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- .github/actions/npm-cache/action.yml | 16 +- .github/workflows/cache-retention.yml | 57 +++ .github/workflows/ci.yml | 8 + docs/testing.md | 41 +- scripts/ci/cache-retention.d.mts | 61 +++ scripts/ci/cache-retention.mjs | 358 ++++++++++++++ .../electronIpcListenerCallsiteCli.test.ts | 10 +- .../boundary/filesystem/ciCachePolicy.test.ts | 181 ++++++- .../filesystem/ciCacheRetention.test.ts | 464 ++++++++++++++++++ 9 files changed, 1172 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/cache-retention.yml create mode 100644 scripts/ci/cache-retention.d.mts create mode 100644 scripts/ci/cache-retention.mjs create mode 100644 tests/boundary/filesystem/ciCacheRetention.test.ts diff --git a/.github/actions/npm-cache/action.yml b/.github/actions/npm-cache/action.yml index 1af22da2b9..dbcade5fd0 100644 --- a/.github/actions/npm-cache/action.yml +++ b/.github/actions/npm-cache/action.yml @@ -1,5 +1,10 @@ name: Cache npm package downloads -description: Restore npm's download cache and save it only from trusted default-branch events. +description: Restore npm's download cache. Trusted default-branch jobs that opt in may save it. +inputs: + save: + description: Save the shared download cache when this event is allowed to write it. + required: false + default: "false" outputs: can-save: description: Whether this run may update the shared npm download cache. @@ -23,20 +28,25 @@ runs: can_save=true fi cache_key="npm-download-${RUNNER_OS}-${runtime_arch}-${LOCKFILES_HASH}" + # Same platform only. A lockfile change can reuse tarballs from the previous key. + restore_key="npm-download-${RUNNER_OS}-${runtime_arch}-" { printf 'can-save=%s\n' "${can_save}" printf 'path=%s\n' "${cache_path}" printf 'key=%s\n' "${cache_key}" + printf 'restore-key=%s\n' "${restore_key}" } >> "${GITHUB_OUTPUT}" - name: Restore and save npm package downloads - if: steps.context.outputs.can-save == 'true' + if: steps.context.outputs.can-save == 'true' && inputs.save == 'true' uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ steps.context.outputs.path }} key: ${{ steps.context.outputs.key }} + restore-keys: ${{ steps.context.outputs.restore-key }} - name: Restore npm package downloads without saving - if: steps.context.outputs.can-save != 'true' + if: steps.context.outputs.can-save != 'true' || inputs.save != 'true' uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ steps.context.outputs.path }} key: ${{ steps.context.outputs.key }} + restore-keys: ${{ steps.context.outputs.restore-key }} diff --git a/.github/workflows/cache-retention.yml b/.github/workflows/cache-retention.yml new file mode 100644 index 0000000000..70545e3ebe --- /dev/null +++ b/.github/workflows/cache-retention.yml @@ -0,0 +1,57 @@ +name: Cache retention + +on: + workflow_run: + workflows: ["CI"] + types: [completed] + +# Qualification calls ci.yml with actions: read. A nested workflow cannot raise +# that token, so cache deletion stays in this workflow. +permissions: + contents: read + actions: write + +concurrency: + group: cache-retention-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: false + +jobs: + retain: + name: Retain Turbo snapshots and npm download caches + # Checkout and run only a same-repository default-branch commit. Pull + # requests, including forks, do not reach these steps. Static and build + # save Turbo snapshots before later lanes finish, so a failed, timed-out, + # or cancelled run still retains those snapshots. + if: >- + (github.event.workflow_run.conclusion == 'success' || + github.event.workflow_run.conclusion == 'failure' || + github.event.workflow_run.conclusion == 'timed_out' || + github.event.workflow_run.conclusion == 'cancelled') && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.head_branch == github.event.repository.default_branch && + (github.event.workflow_run.event == 'push' || + github.event.workflow_run.event == 'schedule' || + github.event.workflow_run.event == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.event.workflow_run.head_sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Retain current Turbo snapshots and npm download caches + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CACHE_PREFIX: rea-static-v1-${{ runner.os }}-${{ runner.arch }}- + CACHE_KEEP_KEY: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }} + CACHE_DEEP_PREFIX: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}- + CACHE_DEEP_KEEP_KEY: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }} + CACHE_KNIP_PREFIX: knip-meta-v1-${{ runner.os }}- + CACHE_KNIP_KEEP_KEY: knip-meta-v1-${{ runner.os }}-${{ hashFiles('knip.json', 'tsconfig.json', 'package.json', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }} + CACHE_PORTABLE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}- + DEDUPE_NPM_DOWNLOADS: "true" + run: node scripts/ci/cache-retention.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a836c18406..4998c8e5db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,9 @@ jobs: - name: Cache npm package downloads id: npm-cache uses: ./.github/actions/npm-cache + with: + # Linux x64 writer. Parallel Linux jobs restore this key. + save: "true" - run: npm ci - name: Restore static task cache if: needs.changes.outputs.code == 'true' @@ -658,6 +661,9 @@ jobs: - name: Cache npm package downloads id: npm-cache uses: ./.github/actions/npm-cache + with: + # ubuntu-latest is the Linux x64 writer above. Other hosts save their own key. + save: ${{ matrix.os != 'ubuntu-latest' }} - name: Verify native runtime identity env: EXPECTED_CI_PLATFORM: ${{ matrix.platform }} @@ -794,6 +800,8 @@ jobs: - name: Cache npm package downloads id: npm-cache uses: ./.github/actions/npm-cache + with: + save: "true" - run: npm ci - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/docs/testing.md b/docs/testing.md index 7dc7705ca1..e3a8131b57 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -579,8 +579,9 @@ metadata, and available manifests into `.cache/ci-build-cache/`. `portable-build-cache.json` records availability per task. Evicted or partially missing entries are omitted as whole entries, and consumers rebuild cache misses, including when every entry is unavailable. Malformed plans, non-file entries, -and I/O errors other than disappearance still fail publication. Older cross-run entries stay in the -original cache. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack +and I/O errors other than disappearance still fail publication. After a trusted +static run, superseded cross-run Turbo snapshots for that OS and architecture +are deleted down to the two newest generations. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack commands, which restore matching outputs or rebuild on a cache miss. Native artifacts, dependency installs, package archives, and runtime verification remain host-local. Keep platform-dependent outputs out of this shared cache. @@ -1370,13 +1371,35 @@ token leaves the local Turbo cache in place. The shared npm-cache action caches downloads, not `node_modules`, and every consumer still runs `npm ci`. Keys cover the runner OS, actual Node architecture, -root lockfiles, and `.npmrc`. Only push, schedule, and manual events on the -repository's default branch may save shared npm caches; PRs restore only. -Static snapshots use the same write authority and are saved only after the -static job's validation succeeds. Main warms the new cache namespace after -merge; a cold cache or changed lockfile falls back to the normal install. -This prevents each PR from saving another branch-scoped copy of the same -download cache. It does not remove runner assignment waits. +root lockfiles, and `.npmrc`. An exact-key miss restores the newest cache for +that OS and architecture, so a lockfile change reuses unchanged tarballs while +`npm ci` still verifies the lockfile. Only push, schedule, and manual events on +the repository's default branch may save shared npm caches, and only from the +opted-in writer for that host: Static checks for Linux x64, the package matrix +for every other included host, and the Windows curated lane. Every other job +restores without saving, so parallel jobs do not publish duplicate copies. +Pull requests restore only. Static snapshots are saved only after the static +job's validation succeeds. Cache deletion is a +separate workflow, because reusable qualification calls this workflow and +cannot raise its read-only token. After a push, schedule, or manual CI run on +the default branch finishes, including when a later lane fails, times out, or +is cancelled, that workflow runs on `ubuntu-latest` and deletes older +`rea-static-v1`, `rea-static-deep-v1`, and `rea-portable-build-v2` entries for +that runner's OS and architecture, plus older `knip-meta-v1` entries for that +OS. It keeps two generations plus the key that run saved. A fast run that did +not publish a deep or Knip snapshot still leaves the two newest entries in +those families. It +deletes an npm download record only when another record has the same key, cache +version, and git ref, and it keeps the most recently accessed copy. A different +version or ref is a different restore target and stays. One cache id is one +record: a repeated id keeps the later access time when its key, version, ref, +size, and creation time agree, and an inconsistent repeat deletes nothing. An +incomplete or unexpected cache list deletes nothing. +Distinct lockfile generations of the download cache stay, so the restore +prefix still has a seed. +Main warms the new cache namespace after merge; a cold cache falls back to the +normal install. This prevents each PR from saving another branch-scoped copy of +the same download cache. It does not remove runner assignment waits. Portable owner-based test selection remains observational. Its reports compare proposed omissions with the full run; incomplete or skipped observations do not diff --git a/scripts/ci/cache-retention.d.mts b/scripts/ci/cache-retention.d.mts new file mode 100644 index 0000000000..d5d02c19c8 --- /dev/null +++ b/scripts/ci/cache-retention.d.mts @@ -0,0 +1,61 @@ +export interface RetainedCacheEntry { + readonly id: number; + readonly key: string; + readonly createdAt: number; + readonly accessedAt: number; + readonly size: number; +} + +export interface CacheListRecord { + readonly id: number; + readonly key: string; + readonly version: string; + readonly ref: string; + readonly created_at: string; + readonly last_accessed_at?: string; + readonly size_in_bytes: number; +} + +export interface RetentionFamily { + readonly prefix: string; + readonly keepKey?: string; +} + +export type CacheRequest = ( + method: string, + endpoint: string, +) => Promise; + +export function assertRetentionPrefix(prefix: string): void; + +export function selectPrefixDeletions( + caches: readonly unknown[], + family?: RetentionFamily, + keepGenerations?: number, +): RetainedCacheEntry[]; + +export function selectNpmDownloadDuplicateDeletions( + caches: readonly unknown[], +): RetainedCacheEntry[]; + +export function listAllCaches( + request: CacheRequest, + repository: string, + prefix: string, +): Promise; + +export function retainCaches(options: { + repository: string; + request: CacheRequest; + deleteCache: (id: number) => Promise; + families?: readonly RetentionFamily[]; + dedupeNpmDownloads?: boolean; + dryRun?: boolean; +}): Promise<{ + dryRun: boolean; + plan: readonly { + prefix: string; + deletions: readonly RetainedCacheEntry[]; + }[]; + npmDeletions: readonly RetainedCacheEntry[]; +}>; diff --git a/scripts/ci/cache-retention.mjs b/scripts/ci/cache-retention.mjs new file mode 100644 index 0000000000..dce987bd86 --- /dev/null +++ b/scripts/ci/cache-retention.mjs @@ -0,0 +1,358 @@ +import { execFile } from "node:child_process"; +import { realpathSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const MAX_BUFFER = 8 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 20_000; +const MAX_PAGES = 10; +const PAGE_SIZE = 100; + +// Keep the newest snapshots, not only the key just saved. A concurrent job may +// still be restoring the previous generation; deleting that id fails its restore. +// The following trusted run drops anything older than those two. +const RETAINED_CACHE_GENERATIONS = 2; + +const PREFIX_PATTERNS = [ + /^rea-static-v1-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u, + /^rea-static-deep-v1-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u, + /^rea-portable-build-v2-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u, + /^knip-meta-v1-[A-Za-z0-9]+-$/u, +]; + +/** Reject prefixes that could match npm downloads or another platform. */ +export function assertRetentionPrefix(prefix) { + if ( + typeof prefix !== "string" || + !PREFIX_PATTERNS.some((pattern) => pattern.test(prefix)) + ) + throw new Error( + "Cache retention prefix must be one Turbo family for one OS and architecture, or one Knip metadata family for one OS", + ); +} + +function normalizeCache(cache, index) { + if (typeof cache !== "object" || cache === null || Array.isArray(cache)) + throw new Error(`Cache entry ${String(index + 1)} is malformed`); + const { id, key, size_in_bytes: size } = cache; + if (!Number.isSafeInteger(id) || id <= 0) + throw new Error(`Cache entry ${String(index + 1)} has an invalid id`); + if (typeof key !== "string" || key.length === 0 || /[\r\n]/u.test(key)) + throw new Error(`Cache entry ${String(index + 1)} has an invalid key`); + const createdAt = Date.parse(cache.created_at); + const accessedAt = Date.parse(cache.last_accessed_at ?? cache.created_at); + if (!Number.isFinite(createdAt) || !Number.isFinite(accessedAt)) + throw new Error( + `Cache entry ${String(index + 1)} has an invalid timestamp`, + ); + if (!Number.isSafeInteger(size) || size < 0) + throw new Error(`Cache entry ${String(index + 1)} has an invalid size`); + // actions/cache restores a key only when this metadata hash matches. + const version = cache.version; + if (typeof version !== "string" || !/^[a-f0-9]{64}$/u.test(version)) + throw new Error(`Cache entry ${String(index + 1)} has an invalid version`); + // The git ref is the cache's access scope. Another branch cannot use it. + const ref = cache.ref; + if (typeof ref !== "string" || !/^refs\/[^\s]+$/u.test(ref)) + throw new Error(`Cache entry ${String(index + 1)} has an invalid ref`); + return { id, key, version, ref, createdAt, accessedAt, size }; +} + +function sameStableIdentity(left, right) { + return ( + left.key === right.key && + left.version === right.version && + left.ref === right.ref && + left.size === right.size && + left.createdAt === right.createdAt + ); +} + +// The list API orders by last access. A restore between pages can place one +// id on two pages. That is still one cache: collapse it, or abort when the +// pages disagree, before any id can be both kept and deleted. +function uniqueById(entries) { + const byId = new Map(); + for (const entry of entries) { + const existing = byId.get(entry.id); + if (existing === undefined) { + byId.set(entry.id, entry); + continue; + } + if (!sameStableIdentity(existing, entry)) + throw new Error( + `Cache id ${String(entry.id)} was listed with inconsistent fields`, + ); + if (entry.accessedAt > existing.accessedAt) byId.set(entry.id, entry); + } + return [...byId.values()]; +} + +function byNewest(left, right) { + if (left.createdAt !== right.createdAt) + return right.createdAt - left.createdAt; + return right.id - left.id; +} + +/** + * Choose Turbo snapshot ids to delete. `keepKey` is protected even when it is + * older than the retained generations. Keys outside the prefix abort retention. + */ +export function selectPrefixDeletions( + caches, + { prefix, keepKey } = {}, + keepGenerations = RETAINED_CACHE_GENERATIONS, +) { + assertRetentionPrefix(prefix); + if (!Number.isSafeInteger(keepGenerations) || keepGenerations < 1) + throw new Error("Cache retention must keep at least one generation"); + if (keepKey !== undefined) { + if ( + typeof keepKey !== "string" || + !keepKey.startsWith(prefix) || + keepKey === prefix + ) + throw new Error("Cache retention keep key must extend its prefix"); + } + const entries = uniqueById(caches.map(normalizeCache)); + for (const entry of entries) + if (!entry.key.startsWith(prefix)) + throw new Error( + `Refusing to evaluate a cache outside ${prefix}: ${entry.key}`, + ); + const protectedIds = new Set( + [...entries] + .sort(byNewest) + .slice(0, keepGenerations) + .map((entry) => entry.id), + ); + if (keepKey !== undefined) + for (const entry of entries) + if (entry.key === keepKey) protectedIds.add(entry.id); + return entries.filter((entry) => !protectedIds.has(entry.id)); +} + +/** + * Drop extra copies of one npm restore target. Records that share a user key + * but differ by cache version or git ref stay, because a workflow can restore + * only a matching version inside its visible scope. + */ +export function selectNpmDownloadDuplicateDeletions(caches) { + const entries = uniqueById(caches.map(normalizeCache)); + for (const entry of entries) + if (!entry.key.startsWith("npm-download-")) + throw new Error( + `Refusing to evaluate a non-download cache: ${entry.key}`, + ); + const groups = new Map(); + for (const entry of entries) { + const identity = `${entry.version}\n${entry.ref}\n${entry.key}`; + const group = groups.get(identity) ?? []; + group.push(entry); + groups.set(identity, group); + } + const deletions = []; + for (const group of groups.values()) { + if (group.length < 2) continue; + const ranked = [...group].sort((left, right) => { + if (left.accessedAt !== right.accessedAt) + return right.accessedAt - left.accessedAt; + if (left.createdAt !== right.createdAt) + return right.createdAt - left.createdAt; + return right.id - left.id; + }); + deletions.push(...ranked.slice(1)); + } + return deletions; +} + +export async function listAllCaches(request, repository, prefix) { + const caches = []; + for (let page = 1; page <= MAX_PAGES; page += 1) { + const body = await request( + "GET", + `repos/${repository}/actions/caches?per_page=${String(PAGE_SIZE)}&page=${String(page)}&key=${encodeURIComponent(prefix)}`, + ); + if ( + typeof body !== "object" || + body === null || + Array.isArray(body) || + !Array.isArray(body.actions_caches) + ) + throw new Error("GitHub cache list is malformed"); + caches.push(...body.actions_caches); + if (body.actions_caches.length < PAGE_SIZE) return caches; + } + throw new Error( + `Cache list for ${prefix} exceeded ${String(MAX_PAGES)} pages; deleted nothing`, + ); +} + +/** + * List every requested family before deleting. A malformed list or an unsafe + * prefix leaves the repository caches unchanged. + */ +export async function retainCaches({ + repository, + request, + deleteCache, + families = [], + dedupeNpmDownloads = false, + dryRun = false, +}) { + if ( + typeof repository !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) + ) + throw new Error( + "CI_REPOSITORY must be an owner/name repository identifier", + ); + for (const family of families) { + assertRetentionPrefix(family.prefix); + if ( + family.keepKey !== undefined && + (typeof family.keepKey !== "string" || + !family.keepKey.startsWith(family.prefix) || + family.keepKey === family.prefix) + ) + throw new Error("Cache retention keep key must extend its prefix"); + } + const plan = []; + for (const family of families) { + const caches = await listAllCaches(request, repository, family.prefix); + plan.push({ + prefix: family.prefix, + deletions: selectPrefixDeletions(caches, family), + }); + } + const npmDeletions = dedupeNpmDownloads + ? selectNpmDownloadDuplicateDeletions( + await listAllCaches(request, repository, "npm-download-"), + ) + : []; + if (!dryRun) + for (const entry of [ + ...plan.flatMap((family) => family.deletions), + ...npmDeletions, + ]) + await deleteCache(entry.id); + return { dryRun, plan, npmDeletions }; +} + +function repositoryFromEnv(env) { + const repository = env.CI_REPOSITORY; + if ( + typeof repository !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) + ) + throw new Error( + "CI_REPOSITORY must be an owner/name repository identifier", + ); + return repository; +} + +function familyWithKeepKey(env, prefixName, keepName) { + const prefix = env[prefixName]; + const keepKey = env[keepName]; + const hasPrefix = typeof prefix === "string" && prefix.length > 0; + const hasKeepKey = typeof keepKey === "string" && keepKey.length > 0; + if (!hasPrefix && !hasKeepKey) return undefined; + if (!hasPrefix || !hasKeepKey) + throw new Error(`${keepName} is required with ${prefixName}`); + return { prefix, keepKey }; +} + +function familiesFromEnv(env) { + const families = [ + familyWithKeepKey(env, "CACHE_PREFIX", "CACHE_KEEP_KEY"), + familyWithKeepKey(env, "CACHE_DEEP_PREFIX", "CACHE_DEEP_KEEP_KEY"), + familyWithKeepKey(env, "CACHE_KNIP_PREFIX", "CACHE_KNIP_KEEP_KEY"), + typeof env.CACHE_PORTABLE_PREFIX === "string" && + env.CACHE_PORTABLE_PREFIX.length > 0 + ? { prefix: env.CACHE_PORTABLE_PREFIX } + : undefined, + ].filter((family) => family !== undefined); + if (families.length === 0 && env.DEDUPE_NPM_DOWNLOADS !== "true") + throw new Error( + "Cache retention requires CACHE_PREFIX or CACHE_PORTABLE_PREFIX or npm dedupe", + ); + return families; +} + +function failureText(error) { + if (typeof error !== "object" || error === null) return String(error); + const stderr = + "stderr" in error && error.stderr !== undefined ? String(error.stderr) : ""; + const message = error instanceof Error ? error.message : ""; + return `${message}\n${stderr}`; +} + +async function githubRequest(method, endpoint, env) { + const { stdout } = await execFileAsync( + "gh", + ["api", "--method", method, endpoint], + { + env, + encoding: "utf8", + maxBuffer: MAX_BUFFER, + timeout: REQUEST_TIMEOUT_MS, + }, + ); + if (method === "DELETE") return undefined; + try { + return JSON.parse(stdout); + } catch (cause) { + throw new Error("GitHub cache API returned malformed JSON", { cause }); + } +} + +async function main() { + const env = process.env; + const dryRun = env.CACHE_RETENTION_DRY_RUN === "true"; + const report = await retainCaches({ + repository: repositoryFromEnv(env), + request: (method, endpoint) => githubRequest(method, endpoint, env), + deleteCache: async (id) => { + try { + await githubRequest( + "DELETE", + `repos/${repositoryFromEnv(env)}/actions/caches/${String(id)}`, + env, + ); + } catch (error) { + if (/404|not found/iu.test(failureText(error))) return; + throw error; + } + }, + families: familiesFromEnv(env), + dedupeNpmDownloads: env.DEDUPE_NPM_DOWNLOADS === "true", + dryRun, + }); + const summary = { + dryRun: report.dryRun, + families: report.plan.map((family) => ({ + prefix: family.prefix, + deleteCount: family.deletions.length, + deleteBytes: family.deletions.reduce((sum, entry) => sum + entry.size, 0), + })), + npmDuplicateDeleteCount: report.npmDeletions.length, + npmDuplicateDeleteBytes: report.npmDeletions.reduce( + (sum, entry) => sum + entry.size, + 0, + ), + }; + process.stdout.write(`${JSON.stringify(summary)}\n`); +} + +function isDirectRun() { + const entry = process.argv[1]; + if (!entry) return false; + try { + return import.meta.url === pathToFileURL(realpathSync(entry)).href; + } catch { + return false; + } +} + +if (isDirectRun()) await main(); diff --git a/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts b/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts index 5d8efcf2a1..54f6883e5d 100644 --- a/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts +++ b/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts @@ -104,12 +104,12 @@ cliTest.for(["CLI", "stdio MCP"] as const)( }; let traceEvidence: unknown; if (surface === "CLI") { + // The application Evidence document exceeds the runner argument limit + // when it is placed on argv. The command reads the same JSON from a file. + const traceInput = join(root, "trace-input.json"); + await writeFile(traceInput, JSON.stringify(traceArguments)); const response = await cli.run({ - arguments: [ - "trace-application-feature", - JSON.stringify(traceArguments), - "--json", - ], + arguments: ["trace-application-feature", traceInput, "--json"], }); expect(response.exitCode, response.stderr).toBe(0); traceEvidence = response.json; diff --git a/tests/boundary/filesystem/ciCachePolicy.test.ts b/tests/boundary/filesystem/ciCachePolicy.test.ts index 6a8867b669..2b70aa82c2 100644 --- a/tests/boundary/filesystem/ciCachePolicy.test.ts +++ b/tests/boundary/filesystem/ciCachePolicy.test.ts @@ -13,6 +13,12 @@ const actionPath = join(repository, ".github/actions/npm-cache/action.yml"); const readAction = async () => z .object({ + inputs: z + .record( + z.string(), + z.object({ default: z.string().optional() }).passthrough(), + ) + .optional(), outputs: z.record(z.string(), z.object({ value: z.string() })), runs: z.object({ steps: z.array( @@ -42,9 +48,11 @@ const readWorkflows = async () => { jobs: z .record( z.string(), - z.object({ - steps: z.array(z.record(z.string(), z.unknown())).optional(), - }), + z + .object({ + steps: z.array(z.record(z.string(), z.unknown())).optional(), + }) + .passthrough(), ) .optional(), }) @@ -107,6 +115,9 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache' expect(output).toContain( `key=npm-download-Windows-${process.arch}-lock-hash\n`, ); + expect(output).toContain( + `restore-key=npm-download-Windows-${process.arch}-\n`, + ); } }); @@ -120,6 +131,7 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache' step.uses?.startsWith("actions/cache/restore@"), ); + expect(action.inputs?.save?.default).toBe("false"); expect(action.outputs["can-save"]?.value).toBe( "${{ steps.context.outputs.can-save }}", ); @@ -138,19 +150,27 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache' expect(restoreAndSave?.uses).toBe( "actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830", ); - expect(restoreAndSave?.if).toBe("steps.context.outputs.can-save == 'true'"); + expect(restoreAndSave?.if).toBe( + "steps.context.outputs.can-save == 'true' && inputs.save == 'true'", + ); expect(restoreOnly?.uses).toBe( "actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830", ); - expect(restoreOnly?.if).toBe("steps.context.outputs.can-save != 'true'"); + expect(restoreOnly?.if).toBe( + "steps.context.outputs.can-save != 'true' || inputs.save != 'true'", + ); expect(restoreAndSave?.with?.path).toBe( "${{ steps.context.outputs.path }}", ); expect(restoreOnly?.with?.path).toBe(restoreAndSave?.with?.path); expect(restoreAndSave?.with?.key).toBe("${{ steps.context.outputs.key }}"); expect(restoreOnly?.with?.key).toBe(restoreAndSave?.with?.key); - expect(restoreAndSave?.with).not.toHaveProperty("restore-keys"); - expect(restoreOnly?.with).not.toHaveProperty("restore-keys"); + expect(restoreAndSave?.with?.["restore-keys"]).toBe( + "${{ steps.context.outputs.restore-key }}", + ); + expect(restoreOnly?.with?.["restore-keys"]).toBe( + restoreAndSave?.with?.["restore-keys"], + ); }); it("replaces every inline setup-node npm cache immediately and preserves conditions", async () => { @@ -258,6 +278,153 @@ it("shares static task snapshots only after validation using the npm cache owner path: "node_modules/.cache/knip", key: "${{ steps.knip-cache.outputs.cache-primary-key }}", }); + expect( + steps.find((step) => step.run === "node scripts/ci/cache-retention.mjs"), + ).toBeUndefined(); +}); + +it("deletes superseded caches only from the trusted follow-up workflow", async () => { + const workflow = z + .object({ + on: z.object({ + workflow_run: z.object({ + workflows: z.array(z.literal("CI")), + types: z.array(z.literal("completed")), + }), + workflow_call: z.unknown().optional(), + pull_request: z.unknown().optional(), + }), + permissions: z.record(z.string(), z.string()), + jobs: z.record( + z.string(), + z.object({ + if: z.string(), + "runs-on": z.literal("ubuntu-latest"), + steps: z.array(z.record(z.string(), z.unknown())), + }), + ), + }) + .parse( + parse( + await readFile( + join(repository, ".github/workflows/cache-retention.yml"), + "utf8", + ), + ), + ); + expect(workflow.on.workflow_call).toBeUndefined(); + expect(workflow.on.pull_request).toBeUndefined(); + expect(workflow.permissions).toEqual({ + contents: "read", + actions: "write", + }); + const job = workflow.jobs.retain; + expect(job?.if).toContain( + "github.event.workflow_run.conclusion == 'success'", + ); + expect(job?.if).toContain( + "github.event.workflow_run.conclusion == 'failure'", + ); + expect(job?.if).toContain( + "github.event.workflow_run.conclusion == 'timed_out'", + ); + expect(job?.if).toContain( + "github.event.workflow_run.conclusion == 'cancelled'", + ); + expect(job?.if).toContain( + "github.event.workflow_run.head_repository.full_name == github.repository", + ); + expect(job?.if).toContain( + "github.event.workflow_run.head_branch == github.event.repository.default_branch", + ); + expect(job?.if).toContain("github.event.workflow_run.event == 'push'"); + expect(job?.if).toContain("github.event.workflow_run.event == 'schedule'"); + expect(job?.if).toContain( + "github.event.workflow_run.event == 'workflow_dispatch'", + ); + expect(job?.if).not.toContain("pull_request"); + const steps = job?.steps ?? []; + expect(steps.some((step) => step.run === "npm ci")).toBe(false); + const checkout = steps[0]; + expect(checkout?.uses).toBe( + "actions/checkout@11d5960a326750d5838078e36cf38b85af677262", + ); + expect(checkout?.with).toMatchObject({ + ref: "${{ github.event.workflow_run.head_sha }}", + "persist-credentials": false, + }); + const retention = steps.find( + (step) => step.run === "node scripts/ci/cache-retention.mjs", + ); + const retentionEnv = z.record(z.string(), z.string()).parse(retention?.env); + expect(retentionEnv.CACHE_PREFIX).toBe( + "rea-static-v1-${{ runner.os }}-${{ runner.arch }}-", + ); + expect(retentionEnv.CACHE_KEEP_KEY).toBe( + "rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}", + ); + expect(retentionEnv.CACHE_PORTABLE_PREFIX).toBe( + "rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-", + ); + expect(retentionEnv.DEDUPE_NPM_DOWNLOADS).toBe("true"); + expect(retentionEnv.GH_TOKEN).toBe("${{ github.token }}"); + const ci = (await readWorkflows()).find(({ file }) => file === "ci.yml"); + expect(JSON.stringify(ci?.workflow)).not.toContain('"actions":"write"'); + const deepSteps = ci?.workflow.jobs?.["static-deep"]?.steps ?? []; + const publishedKey = (id: string) => + z + .string() + .parse( + z + .record(z.string(), z.unknown()) + .parse(deepSteps.find((step) => step.id === id)?.with).key, + ); + const savedOnThisRun = "${{ github.event.workflow_run.head_sha }}"; + const deepPrefix = "rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-"; + const knipPrefix = "knip-meta-v1-${{ runner.os }}-"; + expect(retentionEnv.CACHE_DEEP_PREFIX).toBe(deepPrefix); + expect(retentionEnv.CACHE_DEEP_KEEP_KEY).toBe( + publishedKey("static-cache").replaceAll( + "${{ github.sha }}", + savedOnThisRun, + ), + ); + expect(publishedKey("static-cache").startsWith(deepPrefix)).toBe(true); + expect(retentionEnv.CACHE_KNIP_PREFIX).toBe(knipPrefix); + expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toBe( + publishedKey("knip-cache").replaceAll("${{ github.sha }}", savedOnThisRun), + ); + expect(publishedKey("knip-cache").startsWith(knipPrefix)).toBe(true); +}); + +it("publishes each hosted npm download cache from one writer", async () => { + const workflows = await readWorkflows(); + const writers: { file: string; job: string; save: string }[] = []; + for (const { file, workflow } of workflows) + for (const [job, definition] of Object.entries(workflow.jobs ?? {})) + for (const step of definition.steps ?? []) { + if (step.uses !== "./.github/actions/npm-cache") continue; + const save = z + .record(z.string(), z.unknown()) + .parse(step.with ?? {}).save; + if (save === undefined) continue; + writers.push({ file, job, save: z.string().parse(save) }); + } + expect(writers).toEqual([ + { file: "ci.yml", job: "static", save: "true" }, + { + file: "ci.yml", + job: "package-e2e", + save: "${{ matrix.os != 'ubuntu-latest' }}", + }, + { file: "ci.yml", job: "windows-curated", save: "true" }, + ]); + const ci = workflows.find(({ file }) => file === "ci.yml"); + expect(ci?.workflow.jobs?.static?.permissions).toBeUndefined(); + expect(ci?.workflow.jobs?.build?.permissions).toEqual({ + contents: "read", + actions: "read", + }); }); it("passes the npm cache owner's selected path to package installation", async () => { diff --git a/tests/boundary/filesystem/ciCacheRetention.test.ts b/tests/boundary/filesystem/ciCacheRetention.test.ts new file mode 100644 index 0000000000..d455b83500 --- /dev/null +++ b/tests/boundary/filesystem/ciCacheRetention.test.ts @@ -0,0 +1,464 @@ +import { execFileSync } from "node:child_process"; +import { chmod, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { delimiter, join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + listAllCaches, + retainCaches, + selectNpmDownloadDuplicateDeletions, + selectPrefixDeletions, +} from "../../../scripts/ci/cache-retention.mjs"; + +const cache = ( + id: number, + key: string, + created: string, + accessed = created, + size = 100, + version = "a".repeat(64), + ref = "refs/heads/main", +) => ({ + id, + key, + version, + ref, + created_at: created, + last_accessed_at: accessed, + size_in_bytes: size, +}); + +describe("Turbo snapshot retention", () => { + const prefix = "rea-static-v1-Linux-X64-"; + + it("keeps the two newest snapshots and the saved key", () => { + const keepKey = `${prefix}saved`; + const deletions = selectPrefixDeletions( + [ + cache(1, `${prefix}old`, "2026-10-20T01:00:00Z", undefined, 10), + cache(2, `${prefix}older`, "2026-10-20T02:00:00Z", undefined, 20), + cache(3, `${prefix}recent`, "2026-10-20T03:00:00Z", undefined, 30), + cache(4, keepKey, "2026-10-20T00:00:00Z", undefined, 40), + ], + { prefix, keepKey }, + ); + expect(deletions.map((entry) => entry.id)).toEqual([1]); + expect(deletions.reduce((sum, entry) => sum + entry.size, 0)).toBe(10); + }); + + it("keeps an in-flight generation when the saved key is already newest", () => { + const keepKey = `${prefix}newest`; + const deletions = selectPrefixDeletions( + [ + cache(1, `${prefix}a`, "2026-10-20T01:00:00Z"), + cache(2, `${prefix}b`, "2026-10-20T02:00:00Z"), + cache(3, `${prefix}c`, "2026-10-20T03:00:00Z"), + cache(4, keepKey, "2026-10-20T04:00:00Z"), + ], + { prefix, keepKey }, + ); + expect( + deletions.map((entry) => entry.id).sort((left, right) => left - right), + ).toEqual([1, 2]); + }); + + it("keeps two generations of the deep and Knip snapshot families", () => { + const deep = "rea-static-deep-v1-Linux-X64-"; + const knip = "knip-meta-v1-Linux-"; + expect( + selectPrefixDeletions( + [ + cache(1, `${deep}old`, "2026-10-20T01:00:00Z"), + cache(2, `${deep}mid`, "2026-10-20T02:00:00Z"), + cache(3, `${deep}new`, "2026-10-20T03:00:00Z"), + ], + { prefix: deep, keepKey: `${deep}new` }, + ).map((entry) => entry.id), + ).toEqual([1]); + expect( + selectPrefixDeletions( + [ + cache(4, `${knip}old`, "2026-10-20T01:00:00Z"), + cache(5, `${knip}mid`, "2026-10-20T02:00:00Z"), + cache(6, `${knip}new`, "2026-10-20T03:00:00Z"), + ], + { prefix: knip, keepKey: `${knip}new` }, + ).map((entry) => entry.id), + ).toEqual([4]); + }); + + it("retains the newest portable snapshots without a caller keep key", () => { + const portable = "rea-portable-build-v2-Linux-X64-"; + const deletions = selectPrefixDeletions( + [ + cache(7, `${portable}one`, "2026-10-20T01:00:00Z"), + cache(8, `${portable}two`, "2026-10-20T02:00:00Z"), + cache(9, `${portable}three`, "2026-10-20T03:00:00Z"), + ], + { prefix: portable }, + ); + expect(deletions.map((entry) => entry.id)).toEqual([7]); + }); + + it("refuses prefixes and lists that can reach download caches", () => { + expect(() => + selectPrefixDeletions([], { prefix: "npm-download-Linux-x64-" }), + ).toThrow(/one Turbo family/u); + expect(() => + selectPrefixDeletions([], { prefix: "rea-static-v1-" }), + ).toThrow(/one Turbo family/u); + expect(() => + selectPrefixDeletions([], { prefix: "knip-meta-v1-" }), + ).toThrow(/one Turbo family/u); + expect(() => + selectPrefixDeletions( + [cache(1, "npm-download-Linux-x64-abc", "2026-10-20T01:00:00Z")], + { + prefix, + keepKey: `${prefix}saved`, + }, + ), + ).toThrow(/outside/u); + expect(() => + selectPrefixDeletions([], { + prefix, + keepKey: "rea-static-v1-Windows-X64-other", + }), + ).toThrow(/keep key/u); + }); +}); + +describe("npm download duplicate retention", () => { + it("deletes only an older copy of the same key, version, and ref", () => { + const key = "npm-download-Linux-x64-lock"; + const version = "a".repeat(64); + const ref = "refs/heads/main"; + const deletions = selectNpmDownloadDuplicateDeletions([ + cache( + 1, + key, + "2026-10-20T01:00:00Z", + "2026-10-20T02:00:00Z", + 179, + version, + ref, + ), + cache( + 2, + key, + "2026-10-20T01:05:00Z", + "2026-10-20T03:00:00Z", + 199, + version, + ref, + ), + cache( + 3, + "npm-download-Windows-x64-lock", + "2026-10-20T01:00:00Z", + undefined, + 186, + ), + ]); + expect(deletions.map((entry) => entry.id)).toEqual([1]); + }); + + it("keeps records that share a key but differ by version or ref", () => { + const key = "npm-download-Linux-x64-lock"; + const version = "a".repeat(64); + const otherVersion = "b".repeat(64); + const deletions = selectNpmDownloadDuplicateDeletions([ + cache( + 1, + key, + "2026-10-20T01:00:00Z", + "2026-10-20T04:00:00Z", + 199, + version, + "refs/heads/main", + ), + cache( + 2, + key, + "2026-10-20T01:00:00Z", + "2026-10-20T03:00:00Z", + 188, + otherVersion, + "refs/heads/main", + ), + cache( + 3, + key, + "2026-10-20T01:00:00Z", + "2026-10-20T02:00:00Z", + 180, + version, + "refs/heads/feature", + ), + ]); + expect(deletions).toEqual([]); + }); + + it("refuses a download record without a version or ref", () => { + const key = "npm-download-Linux-x64-lock"; + const { version: _version, ...withoutVersion } = cache( + 1, + key, + "2026-10-20T01:00:00Z", + ); + expect(() => selectNpmDownloadDuplicateDeletions([withoutVersion])).toThrow( + /version/u, + ); + const { ref: _ref, ...withoutRef } = cache(2, key, "2026-10-20T01:00:00Z"); + expect(() => selectNpmDownloadDuplicateDeletions([withoutRef])).toThrow( + /ref/u, + ); + }); + + it("does not delete the only copy when pagination repeats its id", () => { + const key = "npm-download-Linux-x64-lock"; + const version = "a".repeat(64); + const ref = "refs/heads/main"; + const deletions = selectNpmDownloadDuplicateDeletions([ + cache( + 5, + key, + "2026-10-20T01:00:00Z", + "2026-10-20T02:00:00Z", + 179, + version, + ref, + ), + cache( + 5, + key, + "2026-10-20T01:00:00Z", + "2026-10-20T04:00:00Z", + 179, + version, + ref, + ), + ]); + expect(deletions).toEqual([]); + }); + + it("aborts when the same id is listed with a different version", () => { + const key = "npm-download-Linux-x64-lock"; + expect(() => + selectNpmDownloadDuplicateDeletions([ + cache(5, key, "2026-10-20T01:00:00Z", undefined, 179, "a".repeat(64)), + cache(5, key, "2026-10-20T01:00:00Z", undefined, 179, "b".repeat(64)), + ]), + ).toThrow(/inconsistent/u); + }); + + it("refuses a Turbo snapshot in the download list", () => { + expect(() => + selectNpmDownloadDuplicateDeletions([ + cache(1, "rea-static-v1-Linux-X64-sha", "2026-10-20T01:00:00Z"), + ]), + ).toThrow(/non-download/u); + }); +}); + +describe("retention orchestration", () => { + const repository = "morluto/rea"; + const prefix = "rea-static-v1-Linux-X64-"; + + it("deletes nothing when a later list is incomplete", async () => { + const deleted: number[] = []; + const page = Array.from({ length: 100 }, (_, index) => + cache(index + 1, `${prefix}${String(index)}`, "2026-10-20T01:00:00Z"), + ); + await expect( + retainCaches({ + repository, + request: async (_method, endpoint) => { + if (endpoint.includes("npm-download-")) + throw new Error("list failed after the first family"); + return { actions_caches: page }; + }, + deleteCache: async (id) => { + deleted.push(id); + }, + families: [{ prefix, keepKey: `${prefix}0` }], + dedupeNpmDownloads: true, + }), + ).rejects.toThrow(/list failed|exceeded/u); + expect(deleted).toEqual([]); + }); + + it("stops listing once a short page proves the inventory is complete", async () => { + const requests: string[] = []; + const caches = await listAllCaches( + async (_method, endpoint) => { + requests.push(endpoint); + const page = new URL(endpoint, "https://example.test").searchParams.get( + "page", + ); + return { + actions_caches: + page === "1" + ? Array.from({ length: 100 }, (_, index) => + cache( + index + 1, + `${prefix}${String(index)}`, + "2026-10-20T01:00:00Z", + ), + ) + : [cache(101, `${prefix}last`, "2026-10-20T02:00:00Z")], + }; + }, + repository, + prefix, + ); + expect(caches).toHaveLength(101); + expect(requests).toHaveLength(2); + }); + + it("applies every family only after both lists succeed", async () => { + const deleted: number[] = []; + const keepKey = `${prefix}saved`; + const portable = "rea-portable-build-v2-Linux-X64-"; + const result = await retainCaches({ + repository, + request: async (_method, endpoint) => { + const key = new URL(endpoint, "https://example.test").searchParams.get( + "key", + ); + if (key === prefix) + return { + actions_caches: [ + cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"), + cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"), + cache(3, keepKey, "2026-10-20T03:00:00Z"), + ], + }; + if (key === portable) + return { + actions_caches: [ + cache(4, `${portable}old`, "2026-10-20T01:00:00Z"), + cache(5, `${portable}new`, "2026-10-20T02:00:00Z"), + ], + }; + return { + actions_caches: [ + cache( + 6, + "npm-download-Linux-x64-lock", + "2026-10-20T01:00:00Z", + "2026-10-20T01:00:00Z", + ), + cache( + 7, + "npm-download-Linux-x64-lock", + "2026-10-20T02:00:00Z", + "2026-10-20T03:00:00Z", + ), + ], + }; + }, + deleteCache: async (id) => { + deleted.push(id); + }, + families: [{ prefix, keepKey }, { prefix: portable }], + dedupeNpmDownloads: true, + }); + expect(deleted).toEqual([1, 6]); + expect(result.plan[0]?.deletions.map((entry) => entry.id)).toEqual([1]); + expect(result.plan[1]?.deletions).toEqual([]); + expect(result.npmDeletions.map((entry) => entry.id)).toEqual([6]); + }); + + it("dry-run reports the same deletions without calling delete", async () => { + const result = await retainCaches({ + repository, + request: async () => ({ + actions_caches: [ + cache(1, `${prefix}old`, "2026-10-20T01:00:00Z", undefined, 50), + cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z", undefined, 60), + cache(3, `${prefix}new`, "2026-10-20T03:00:00Z", undefined, 70), + ], + }), + deleteCache: async () => { + throw new Error("dry-run must not delete"); + }, + families: [{ prefix, keepKey: `${prefix}new` }], + dryRun: true, + }); + expect(result.dryRun).toBe(true); + expect(result.plan[0]?.deletions.map((entry) => entry.id)).toEqual([1]); + }); +}); + +describe("repeated cache ids", () => { + const repository = "morluto/rea"; + const prefix = "rea-static-v1-Linux-X64-"; + + it("treats one cache id on two pages as a single snapshot", () => { + const keepKey = `${prefix}saved`; + const deletions = selectPrefixDeletions( + [ + cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"), + cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"), + cache(3, keepKey, "2026-10-20T03:00:00Z", "2026-10-20T03:00:00Z"), + cache(3, keepKey, "2026-10-20T03:00:00Z", "2026-10-20T04:00:00Z"), + ], + { prefix, keepKey }, + ); + expect(deletions.map((entry) => entry.id)).toEqual([1]); + }); + + it("does not delete when one cache id is listed with inconsistent fields", async () => { + const deleted: number[] = []; + const repeated = cache(3, `${prefix}new`, "2026-10-20T03:00:00Z"); + await expect( + retainCaches({ + repository, + request: async () => ({ + actions_caches: [ + cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"), + cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"), + repeated, + { ...repeated, version: "b".repeat(64) }, + ], + }), + deleteCache: async (id) => { + deleted.push(id); + }, + families: [{ prefix, keepKey: `${prefix}new` }], + }), + ).rejects.toThrow(/inconsistent/u); + expect(deleted).toEqual([]); + }); +}); + +it("runs retention from the command line without deleting on an unsafe prefix", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-cache-retention-")); + const log = join(root, "gh.log"); + const gh = join(root, "gh"); + await writeFile( + gh, + `#!/bin/sh +printf '%s\\n' "$@" >> "${log}" +exit 1 +`, + ); + await chmod(gh, 0o755); + expect(() => + execFileSync("node", ["scripts/ci/cache-retention.mjs"], { + cwd: process.cwd(), + env: { + ...process.env, + PATH: `${root}${delimiter}${process.env.PATH ?? ""}`, + CI_REPOSITORY: "morluto/rea", + CACHE_PREFIX: "npm-download-Linux-x64-", + CACHE_KEEP_KEY: "npm-download-Linux-x64-lock", + }, + encoding: "utf8", + }), + ).toThrow(/one Turbo family/u); + await expect(readFile(log, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); +}); From 1bfa60ac20e1a344bcf3132c08d15b6860b05c61 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:50:14 +0800 Subject: [PATCH 06/35] fix(cli): preserve caller-selected JSON paths Keep POSIX symlink-parent traversal at the shared JSON input boundary and forward the selected browser executable without a second lexical normalization. Verify Electron, browser, and Evidence-file consumers through registered commands and the built CLI. Refs #5103 --- src/browser/PlaywrightScenarioBrowser.test.ts | 46 +++- src/browser/PlaywrightScenarioBrowser.ts | 4 +- src/cli/electronCommands.ts | 23 +- src/cliBrowserScenarioCommands.ts | 16 +- src/cliJsonInput.ts | 4 +- src/cliJsonPaths.test.ts | 48 +++- .../acceptance/analysisViewCallerPath.test.ts | 62 +++++ tests/boundary/cli/cliJsonCallerPath.test.ts | 232 ++++++++++++++++++ 8 files changed, 422 insertions(+), 13 deletions(-) create mode 100644 tests/acceptance/analysisViewCallerPath.test.ts create mode 100644 tests/boundary/cli/cliJsonCallerPath.test.ts diff --git a/src/browser/PlaywrightScenarioBrowser.test.ts b/src/browser/PlaywrightScenarioBrowser.test.ts index d067674ec7..3bf1e3cb5e 100644 --- a/src/browser/PlaywrightScenarioBrowser.test.ts +++ b/src/browser/PlaywrightScenarioBrowser.test.ts @@ -3,7 +3,7 @@ import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { afterEach, expect, it, vi } from "vitest"; -import { mkdir, writeFile, rm, access } from "node:fs/promises"; +import { mkdir, writeFile, rm, access, symlink } from "node:fs/promises"; import { join } from "node:path"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import type { @@ -14,6 +14,7 @@ import type { } from "playwright-core"; import { browserScenarioSchema } from "../domain/browserScenario.js"; +import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { attachedScenarioLeaseKey, failBrowserScenarioOperation, @@ -91,6 +92,49 @@ it("passes the selected environment to the actual browser launch boundary", asyn }); }); +it.skipIf(process.platform === "win32")( + "passes the caller-selected symlink-parent executable to the launch boundary", + async () => { + const root = await createTestTempDirectory("rea-browser-caller-path-"); + try { + await mkdir(join(root, "real/inner"), { recursive: true }); + await symlink(join(root, "real/inner"), join(root, "link")); + const selected = absoluteCallerPath("link/../browser", { cwd: root }); + let launchedExecutable: string | undefined; + const scenario = browserScenarioSchema.parse({ + browser: { mode: "launch", executable_path: selected }, + start_url: { url: "https://example.test" }, + actions: [ + { step_id: "settle", action: "wait_for_timeout", duration_ms: 1 }, + ], + }); + + await expect( + openPlaywrightScenarioBrowser( + scenario, + {}, + { + launcher: { + connectOverCDP: async () => { + throw new Error("Unexpected connection"); + }, + launchPersistentContext: async (_profile, options) => { + launchedExecutable = options?.executablePath; + throw new Error("fake launch boundary reached"); + }, + }, + }, + ), + ).rejects.toThrow("fake launch boundary reached"); + + expect(launchedExecutable).toBe(selected); + expect(selected).toBe(`${root}/link/../browser`); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, +); + it("keeps Termux execution support when launching Chromium on Android", async () => { Object.defineProperty(process, "platform", { value: "android" }); vi.stubEnv("PREFIX", "/data/data/com.termux/files/usr"); diff --git a/src/browser/PlaywrightScenarioBrowser.ts b/src/browser/PlaywrightScenarioBrowser.ts index bc4f4db26e..ff17ad062f 100644 --- a/src/browser/PlaywrightScenarioBrowser.ts +++ b/src/browser/PlaywrightScenarioBrowser.ts @@ -7,7 +7,7 @@ import { } from "../artifacts/AnalysisCleanupLease.js"; import { chmod, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import type { Browser, @@ -717,7 +717,7 @@ export const openPlaywrightScenarioBrowser = async ( try { await chmod(profilePath, 0o700); context = await launcher.launchPersistentContext(profilePath, { - executablePath: resolve(scenario.browser.executable_path), + executablePath: scenario.browser.executable_path, headless: scenario.browser.headless, acceptDownloads: false, viewport: { diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index 9995a4e8cc..9490b5d9f7 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -24,16 +24,22 @@ import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnal import { withCommandCancellation } from "./commandCancellation.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import type { ElectronActiveObservationPort } from "../application/javascript/ElectronActiveObservationPort.js"; /** Register CLI equivalents of the Electron MCP tools. */ export const registerElectronCommands = ( cli: ReturnType, logger: Logger, environment: Readonly>, + options: { + readonly createActiveProvider?: ( + environment: Readonly>, + ) => ElectronActiveObservationPort; + } = {}, ): void => { registerElectronObservationCommands(cli, logger); registerElectronFuseCommand(cli, logger); - registerElectronActiveCommand(cli, logger, environment); + registerElectronActiveCommand(cli, logger, environment, options); registerJavaScriptApplicationCommand(cli, logger); registerJavaScriptRuntimeReconciliationCommand(cli, logger); }; @@ -86,6 +92,11 @@ const registerElectronActiveCommand = ( cli: ReturnType, logger: Logger, environment: Readonly>, + options: { + readonly createActiveProvider?: ( + environment: Readonly>, + ) => ElectronActiveObservationPort; + }, ): void => { cli.command(CLI_COMMANDS.captureElectronScenario, { description: "Run one owned Electron scenario with cancellable actions", @@ -122,9 +133,13 @@ const registerElectronActiveCommand = ( ); const { captureElectronScenario } = await import("../application/javascript/ElectronActiveObservationService.js"); - const { createElectronScenarioProvider } = - await import("../composition/electronScenario.js"); - const provider = createElectronScenarioProvider(environment); + const provider = + options.createActiveProvider === undefined + ? await import("../composition/electronScenario.js").then( + ({ createElectronScenarioProvider }) => + createElectronScenarioProvider(environment), + ) + : options.createActiveProvider(environment); const result = await finishAnalysisCommand( () => captureElectronScenario(provider, parsed.data, { signal }), { diff --git a/src/cliBrowserScenarioCommands.ts b/src/cliBrowserScenarioCommands.ts index a18ffe2204..737c621d86 100644 --- a/src/cliBrowserScenarioCommands.ts +++ b/src/cliBrowserScenarioCommands.ts @@ -11,6 +11,7 @@ import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; import { BrowserObservationError } from "./domain/browserObservationError.js"; import { analysisInputErrorFromIssues } from "./domain/inputIssueProjection.js"; import { browserScenarioSchema } from "./domain/browserScenario.js"; +import type { BrowserScenarioCapturePort } from "./application/BrowserScenarioCapturePort.js"; import type { JsonValue } from "./domain/jsonValue.js"; import type { Logger } from "pino"; @@ -21,6 +22,11 @@ export const registerBrowserScenarioCommands = ( cli: ReturnType, logger: Logger, environment: Readonly>, + options: { + readonly createProvider?: ( + environment: Readonly>, + ) => BrowserScenarioCapturePort; + } = {}, ): void => { cli.command(CLI_COMMANDS.captureBrowserScenario, { description: @@ -48,9 +54,13 @@ export const registerBrowserScenarioCommands = ( ), ); // Load Playwright only for this command, not during CLI startup. - const { createBrowserScenarioProvider } = - await import("./composition/browserScenario.js"); - const provider = createBrowserScenarioProvider(environment); + const provider = + options.createProvider === undefined + ? await import("./composition/browserScenario.js").then( + ({ createBrowserScenarioProvider }) => + createBrowserScenarioProvider(environment), + ) + : options.createProvider(environment); let result: | Awaited> | undefined; diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.ts index 1f32d262a5..24c54055e0 100644 --- a/src/cliJsonInput.ts +++ b/src/cliJsonInput.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import { absoluteCallerPath } from "./filesystem/callerPath.js"; import { sealOwnedJsonInput } from "./application/OwnedJsonInput.js"; import { readJsonInputFile } from "./application/JsonInputFile.js"; @@ -141,7 +141,7 @@ const resolveJsonPath = ( typeof entry === "string" && entry.trim().length > 0 && !URL_REFERENCE.test(entry) - ? resolve(entry) + ? absoluteCallerPath(entry) : entry, ]; return [key, resolveJsonPath(entry, tail)]; diff --git a/src/cliJsonPaths.test.ts b/src/cliJsonPaths.test.ts index 1486ed2a62..f5bb133380 100644 --- a/src/cliJsonPaths.test.ts +++ b/src/cliJsonPaths.test.ts @@ -1,4 +1,6 @@ -import { resolve } from "node:path"; +import { mkdir, mkdtemp, rm, symlink } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; import { describe, expect, it } from "vitest"; @@ -57,6 +59,50 @@ describe("CLI JSON path normalization", () => { expect(input.browser.executable_path).toBe(" browser "); }); + it.skipIf(process.platform === "win32")( + "keeps POSIX symlink-parent traversal for each selected JSON path", + async () => { + const root = await mkdtemp(join(tmpdir(), "rea-json-caller-path-")); + const originalCwd = process.cwd(); + try { + await mkdir(join(root, "real/inner"), { recursive: true }); + await symlink(join(root, "real/inner"), join(root, "link")); + process.chdir(root); + const electronInput = { + executable_path: "link/../tool", + application_path: "link/../app", + application_root: "link/../root", + }; + const normalizedElectron = resolveCliJsonPaths(electronInput, paths); + expect(normalizedElectron).toEqual({ + executable_path: `${root}/link/../tool`, + application_path: `${root}/link/../app`, + application_root: `${root}/link/../root`, + }); + expect(electronInput.executable_path).toBe("link/../tool"); + expect( + electronActiveObservationInputSchema.safeParse(normalizedElectron) + .success, + ).toBe(true); + const browserInput = { + browser: { executable_path: "link/../browser", label: "keep" }, + }; + expect( + resolveCliJsonPaths(browserInput, [["browser", "executable_path"]]), + ).toEqual({ + browser: { + executable_path: `${root}/link/../browser`, + label: "keep", + }, + }); + expect(browserInput.browser.executable_path).toBe("link/../browser"); + } finally { + process.chdir(originalCwd); + await rm(root, { recursive: true, force: true }); + } + }, + ); + it.each([ "file:///opt/electron", "FILE:electron", diff --git a/tests/acceptance/analysisViewCallerPath.test.ts b/tests/acceptance/analysisViewCallerPath.test.ts new file mode 100644 index 0000000000..ef19e38abf --- /dev/null +++ b/tests/acceptance/analysisViewCallerPath.test.ts @@ -0,0 +1,62 @@ +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { analysisViewResultSchema } from "../../src/domain/analysisView/analysisView.js"; +import { + analysisViewJavaScriptEvidence, + analysisViewLayoutEvidence, +} from "../fixtures/analysisView.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest + .skipIf(process.platform === "win32") + .for(["relative", "absolute"] as const)( + "reads selected analysis-view Evidence through a %s caller path containing a symlink parent", + async (pathKind, { cli }) => { + const root = await createTestTempDirectory( + "rea-analysis-view-caller-path-", + ); + await mkdir(join(root, "real", "inner"), { recursive: true }); + await symlink("real/inner", join(root, "link")); + + const selected = analysisViewJavaScriptEvidence(); + const decoy = analysisViewLayoutEvidence(); + expect(selected.evidence_id).not.toBe(decoy.evidence_id); + await writeFile( + join(root, "real", "evidence.json"), + JSON.stringify(selected), + ); + await writeFile(join(root, "evidence.json"), JSON.stringify(decoy)); + + const sourcePath = + pathKind === "relative" + ? "link/../evidence.json" + : `${root}/link/../evidence.json`; + const queryPath = join(root, "query.json"); + await writeFile( + queryPath, + JSON.stringify({ + source: { kind: "evidence-file", path: sourcePath }, + view: { kind: "summary" }, + }), + ); + + const result = await cli.run({ + arguments: ["inspect-analysis-view", queryPath, "--json"], + cwd: root, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + + const contractResult = toolContract( + "inspect_analysis_view", + ).outputSchema.parse(result.json); + const output = parseEvidence(contractResult); + const view = analysisViewResultSchema.parse(output.normalized_result); + expect(view.kind).toBe("summary"); + expect(view.parent_evidence_id).toBe(selected.evidence_id); + expect(output.evidence_links).toEqual([selected.evidence_id]); + }, +); diff --git a/tests/boundary/cli/cliJsonCallerPath.test.ts b/tests/boundary/cli/cliJsonCallerPath.test.ts new file mode 100644 index 0000000000..b52fddc736 --- /dev/null +++ b/tests/boundary/cli/cliJsonCallerPath.test.ts @@ -0,0 +1,232 @@ +import { + chmod, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Cli } from "incur"; +import pino from "pino"; +import { expect, it } from "vitest"; + +import type { BrowserScenarioCapturePort } from "../../../src/application/BrowserScenarioCapturePort.js"; +import type { ElectronActiveObservationPort } from "../../../src/application/javascript/ElectronActiveObservationPort.js"; +import { registerBrowserScenarioCommands } from "../../../src/cliBrowserScenarioCommands.js"; +import { registerAnalysisViewCommands } from "../../../src/cli/analysisViewCommands.js"; +import { registerElectronCommands } from "../../../src/cli/electronCommands.js"; +import { AnalysisCapabilityUnavailableError } from "../../../src/domain/analysisErrorCore.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { err } from "../../../src/domain/result.js"; +import { + analysisViewJavaScriptEvidence, + analysisViewLayoutEvidence, +} from "../../fixtures/analysisView.js"; + +const providerIdentity = { + id: "test.path-boundary", + name: "Path boundary test provider", + version: null, +} as const; + +const serveJson = async ( + cli: ReturnType, + command: string, + input: unknown, +): Promise => { + let stdout = ""; + await cli.serve([command, JSON.stringify(input), "--json"], { + env: {}, + stdout: (text) => { + stdout += text; + }, + exit: () => undefined, + }); + return JSON.parse(stdout) as unknown; +}; + +it.skipIf(process.platform === "win32")( + "keeps caller-selected JSON paths through the public CLI and provider boundaries", + async () => { + const root = await mkdtemp(join(tmpdir(), "rea-cli-json-path-")); + const originalCwd = process.cwd(); + const originalExitCode = process.exitCode; + try { + await mkdir(join(root, "real/inner"), { recursive: true }); + await mkdir(join(root, "real/app"), { recursive: true }); + await mkdir(join(root, "real/root"), { recursive: true }); + await mkdir(join(root, "app"), { recursive: true }); + await mkdir(join(root, "root"), { recursive: true }); + await writeFile(join(root, "real/tool"), "selected executable"); + await writeFile(join(root, "tool"), "lexical sibling"); + await chmod(join(root, "real/tool"), 0o700); + await chmod(join(root, "tool"), 0o700); + await symlink(join(root, "real/inner"), join(root, "link")); + process.chdir(root); + + const electronInputs: string[][] = []; + const electronProvider: ElectronActiveObservationPort = { + identity: () => providerIdentity, + capture: async (input) => { + electronInputs.push([ + input.executable_path, + input.application_path, + input.application_root, + ]); + return err( + new AnalysisCapabilityUnavailableError( + providerIdentity.id, + "capture_electron_scenario", + "intentional inert test provider", + ), + ); + }, + close: async () => undefined, + }; + const electronCli = Cli.create("rea-path-electron-test", { + sync: false, + }); + registerElectronCommands( + electronCli, + pino({ level: "silent" }), + {}, + { + createActiveProvider: () => electronProvider, + }, + ); + for (const path of ["link/../", `${root}/link/../`]) { + const output = await serveJson( + electronCli, + "capture-electron-scenario", + { + executable_path: `${path}tool`, + application_path: `${path}app`, + application_root: `${path}root`, + actions: [], + }, + ); + expect(output).toMatchObject({ code: "capability_unavailable" }); + } + const expectedElectronPaths = [ + await realpath(join(root, "real/tool")), + await realpath(join(root, "real/app")), + await realpath(join(root, "real/root")), + ]; + expect(electronInputs).toEqual([ + expectedElectronPaths, + expectedElectronPaths, + ]); + + const browserInputs: string[] = []; + const browserProvider: BrowserScenarioCapturePort = { + identity: () => providerIdentity, + captureScenario: async (scenario) => { + if (scenario.browser.mode === "launch") + browserInputs.push(scenario.browser.executable_path); + return err( + new AnalysisCapabilityUnavailableError( + providerIdentity.id, + "capture_browser_scenario", + "intentional inert test provider", + ), + ); + }, + close: async () => undefined, + }; + const browserCli = Cli.create("rea-path-browser-test", { + sync: false, + }); + registerBrowserScenarioCommands( + browserCli, + pino({ level: "silent" }), + {}, + { createProvider: () => browserProvider }, + ); + for (const executablePath of [ + "link/../browser", + `${root}/link/../browser`, + ]) { + const output = await serveJson(browserCli, "capture-browser-scenario", { + browser: { mode: "launch", executable_path: executablePath }, + start_url: { url: "https://example.test" }, + actions: [ + { + step_id: "settle", + action: "wait_for_timeout", + duration_ms: 1, + }, + ], + }); + expect(output).toMatchObject({ code: "capability_unavailable" }); + } + expect(browserInputs).toEqual([ + `${root}/link/../browser`, + `${root}/link/../browser`, + ]); + } finally { + process.chdir(originalCwd); + process.exitCode = originalExitCode; + await rm(root, { recursive: true, force: true }); + } + }, +); + +it.skipIf(process.platform === "win32")( + "reads the caller-selected Evidence file through the registered CLI", + async () => { + const root = await mkdtemp(join(tmpdir(), "rea-cli-evidence-path-")); + const originalCwd = process.cwd(); + const originalExitCode = process.exitCode; + try { + await mkdir(join(root, "real/inner"), { recursive: true }); + await symlink(join(root, "real/inner"), join(root, "link")); + process.chdir(root); + const selectedEvidence = analysisViewLayoutEvidence(); + const siblingEvidence = analysisViewJavaScriptEvidence(); + await mkdir(join(root, "real"), { recursive: true }); + await writeFile( + join(root, "real/evidence.json"), + JSON.stringify(selectedEvidence), + ); + await writeFile( + join(root, "evidence.json"), + JSON.stringify(siblingEvidence), + ); + expect( + parseEvidence( + JSON.parse(await readFile(`${root}/link/../evidence.json`, "utf8")), + ).evidence_id, + ).toBe(selectedEvidence.evidence_id); + const analysisViewCli = Cli.create("rea-path-evidence-test", { + sync: false, + }); + registerAnalysisViewCommands(analysisViewCli, pino({ level: "silent" })); + for (const path of [ + "link/../evidence.json", + `${root}/link/../evidence.json`, + ]) { + const output = parseEvidence( + await serveJson(analysisViewCli, "inspect-analysis-view", { + source: { kind: "evidence-file", path }, + view: { kind: "summary" }, + }), + ); + expect(output.parameters.parent_evidence_id, path).toBe( + selectedEvidence.evidence_id, + ); + expect(output.evidence_links, path).toEqual([ + selectedEvidence.evidence_id, + ]); + } + } finally { + process.chdir(originalCwd); + process.exitCode = originalExitCode; + await rm(root, { recursive: true, force: true }); + } + }, +); From c0cbbf8d506a02d1723aa507224c9a4342d3f015 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:55:40 +0800 Subject: [PATCH 07/35] perf(ci): keep fast built shards under ten minutes (#5652) * test(acceptance): split competing loader ownership across files The loader matrix was one 578-second file, which set the floor for any shard that received it. Three files keep every CLI and MCP case and let workers overlap them. Co-authored-by: morluto * perf(ci): run eight fast built shards under a ten-minute budget Charge each Vitest project by its resolved worker width so serial process-boundary files spread as tail time. Fast feedback uses eight built shards. The npm download cache stays; node_modules is too large to ship as a shared artifact. Co-authored-by: morluto * fix(ci): keep each scheduled file on one worker lane Dividing a file by maxWorkers dropped its own wall-time floor and could pack a parallel file onto a serial tail. Place each file whole on a lane, overlap peers only inside a sequence group, and add later groups after. Co-authored-by: morluto * docs(testing): separate the shard budget from workflow time The ten-minute figure is the built shard job. Classification, build, and the portable-test merge are measured separately on run 39451548945. Co-authored-by: morluto * docs(testing): match fast shard count to lane scheduling Main still described five shards and half-duration weights. Fast qualification uses eight shards, and each file keeps its full duration on one worker. Co-authored-by: morluto * fix(ci): seed split loader durations from pre-split history Timings published before the file split still name one 578-second path. Until a new file appears in that artifact, share the recorded duration by the pre-split case counts. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- .github/workflows/ci.yml | 8 +- docs/testing.md | 60 ++- .../javascriptLiteralLoaderOwnership.ts | 401 ++++++++++++++++++ ...ascriptLiteralLoaderOwnershipEarly.test.ts | 29 ++ ...vascriptLiteralLoaderOwnershipLate.test.ts | 18 + ...scriptLiteralLoaderOwnershipMiddle.test.ts | 18 + .../javascriptLiteralPercentPaths.test.ts | 385 ----------------- .../filesystem/ciTestSequencer.test.ts | 129 +++++- tests/ciTestSequencer.ts | 199 +++++++-- 9 files changed, 779 insertions(+), 468 deletions(-) create mode 100644 tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts create mode 100644 tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts create mode 100644 tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts create mode 100644 tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4998c8e5db..235092e96f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -302,9 +302,11 @@ jobs: # every shard's diagnostics and coverage. Passing PRs run the full suite. fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }} matrix: - # Fast feedback spends the same six test jobs on one source group and - # five built groups; full qualification retains six complete shards. - shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }} + # Fast feedback uses one source group and eight built groups so the + # slowest built shard job stays inside a ten-minute execution budget. + # Classification, build, and report merge sit outside that job. + # Full qualification retains six complete shards with coverage. + shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/8","2/8","3/8","4/8","5/8","6/8","7/8","8/8"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 diff --git a/docs/testing.md b/docs/testing.md index e3a8131b57..51f3f65ce0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -471,8 +471,8 @@ partial diagnostics and preflights do not establish complete mutation outcomes or qualify the release revision for mutation coverage. Fast CI starts the source-only projects after classification, independently of -build and static checks, and runs five built-dependent shards after the build. -The six test jobs still select every portable test. Full qualification keeps six +build and static checks, and runs eight built-dependent shards after the build. +Those nine test jobs still select every portable test. Full qualification keeps six complete Vitest shards with coverage. Neither tier retries tests. Canonical project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest, CI group execution, and development-test preparation. A test importing compiled @@ -533,11 +533,35 @@ JSON artifact members without executing artifact content. Reports and limits are retained in `ci-health-diagnostics`; its ten-minute job limit covers the existing four-minute timing collection plus the bounded diagnostic stages. -Shard-count reduction remains deferred: the available successful timing sample -covers an older inventory, and file phase weights are not job wall time. The -one-source/five-built split retains six test jobs; it makes no hosted speedup -claim. Current complete timings and completed candidate hosted runs are needed -before enabling a smaller adaptive matrix. +Fast built shards target under ten minutes of uncontended shard-job execution. +That figure is the `test-shard` job. The job waits for classification and the +build, and the portable-test merge runs after the slowest shard. On 2026-10-20, +run `39451548945` measured those surrounding jobs with runner queue excluded: +classification 15 seconds, build 49 seconds, and the portable-test merge 29 +seconds. Shard 1/5 of that run spent 22 seconds before built Vitest and 3 +seconds after it, including `npm ci` at 9 seconds. `node_modules` is about 674 +MiB, so publishing it as a shared artifact would transfer more data than that +install saves. The npm download cache stays the setup cache. + +File timings from that run put the built suite at 7944 seconds of module time. +The sequencer keeps each file's full duration on one worker. Extra workers +overlap other files in the same sequence group, and `process-boundary` runs +after that group. Scheduling the unsplit inventory onto five shards peaks at +820 seconds of tests. That run's slowest built shard job lasted 925 seconds +(shard 4/5), and shard 1/5 spent 874 seconds in built Vitest, so hosted job +time sits above the file schedule. Six shards of the same inventory peak at +684 seconds of tests. Eight shards, after the 578-second loader file is split +by case count (54 cases stay in that file and 46 move into each of three +files), peak at 513 seconds of tests. With the measured 25-second shard setup, +that shard job is about 538 seconds. Adding the measured classification, build, +and merge makes an uncontended portable critical path of about 631 seconds, +still before the gap between this file schedule and hosted Vitest. Runner queue +is separate and grows when more shard jobs compete for runners. +`process-boundary` stays serial because process-tree sampling shares the host. +Raising the worker cap from two to four on this four-core host cut a six-file +acceptance sample from 96–101 seconds to 71 seconds and did not remove the +single-file floor, so the cap stays two. Full qualification keeps six covered +shards; its longer instrumentation is outside this fast budget. The separate `CI timings` workflow batches recent completed CI and Release runs hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and @@ -1130,7 +1154,7 @@ suite, without the source-test shards or native package lanes. Classification compares the PR head with its merge base, so later base-branch changes do not expand that scope. Static and portable-test aggregate jobs remain present and fail if classification fails. Every code change retains the -complete six-shard Linux suite; individual test files are not selected by +complete portable Linux suite; individual test files are not selected by imports or filenames. Ordinary PRs also produce an **observational** test-selection plan. The candidate @@ -1141,7 +1165,7 @@ tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall back to the complete inventory. Renames account for both paths. Main, scheduled, manual, and `ci:full` runs always propose the full suite. -All six shards still execute the complete suite. The existing report merge +The source group and every built shard together still execute the complete suite. The existing report merge compares the proposal with actual module outcomes and retains `.cache/ci-test-selection/report.json` in `test-results`, including selected and omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and @@ -1223,7 +1247,7 @@ push: changes from failed or cancelled intermediate runs remain in scope. Missing, untrusted, or non-ancestor history selects the fast Linux x64 baseline: portable checks and the Linux runtime lane. The lookup is bounded to the last 100 successful push runs; lookup failures broaden selection. -Every code change still selects all six portable shards, without coverage +Every code change still selects the source group and all eight fast built shards, without coverage instrumentation in this fast tier. Ordinary PR matrices cancel remaining shards after the first failure; passing PRs execute the complete suite. Main, `ci:full`, scheduled, manual, and release qualification retain every shard's @@ -1336,13 +1360,15 @@ producer; newer schedules that reuse qualification without producing timings do not hide the last full run's artifact. CI assigns discovered files by historical duration, with a deterministic fallback weight for new files, rather than dividing only by file count. -`process-boundary` files are weighted at their full duration because that -project runs files serially. Other projects are weighted at half their file -duration, matching the two-worker cap, so a serial file does not look as cheap -as a file that overlaps with another. Every -discovered file still belongs to exactly one shard; worker limits and project -ordering remain unchanged. Fast qualification keeps five built shards without -coverage. The merge job rejects successful reports with missing, +Each file keeps its full duration on one worker. Extra workers overlap other +files in the same sequence group, and `process-boundary` runs after that +group. History published before the competing-loader split still names only +`javascriptLiteralPercentPaths.test.ts`. Until that artifact records one of +the three new files, the sequencer assigns that recorded duration by the +pre-split case counts: 54/192 to the remaining file and 46/192 to each new +file. Every discovered file still belongs to exactly one shard; worker limits +and project ordering remain unchanged. Fast qualification keeps eight built +shards without coverage. The merge job rejects successful reports with missing, unexpected, or duplicate portable files. Coverage instrumentation and aggregate thresholds remain enforced in full qualification. Missing or unusable history falls back to Vitest's default assignment. Only diff --git a/tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts b/tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts new file mode 100644 index 0000000000..7f9e6ae520 --- /dev/null +++ b/tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts @@ -0,0 +1,401 @@ +import { expect } from "vitest"; + +import { aliasSelectionBudgetSource } from "../../fixtures/javascriptAliasSelectionBudget.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import type { TestCli } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const fixtureImports = (loader: string) => { + switch (loader) { + case "pattern alias": + return { "#window/*": "./proxy/*.cjs" }; + case "recursive alias": + return { "#entry": "#window", "#window": "./proxy/window.cjs" }; + case "external-package alias": + return { "#entry": "fixture-window" }; + case "builtin alias": + return { "#entry": "fs" }; + case "electron alias": + return { "#entry": "electron" }; + case "electron main alias": + return { "#entry": "electron/main" }; + default: + return { "#entry": null }; + } +}; + +const unprovenLoaderSources: Readonly> = { + "custom iterator assignment module loader": + 'const values = { *[Symbol.iterator]() { yield module; } }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");', + "replaced iterator assignment module loader": + 'const values = [{}]; values[Symbol.iterator] = function* () { yield module; }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");', + "custom iterator rest declaration module loader": + 'const values = { *[Symbol.iterator]() { yield module; } }; const [...holder] = values; holder[0].require("../shared/window.cjs");', + "native array assignment local loader": + 'const holder = { loader: module }; [holder.loader] = [{ require() {} }]; holder.loader.require("../shared/window.cjs");', + "logical rewrite assignment module loader": + 'const holder = {}; holder.loader ??= module; holder.loader.require = replacement; require("../shared/window.cjs");', + "object rest declaration module loader": + 'const { ...holder } = { loader: module }; holder.loader.require("../shared/window.cjs");', + "array rest declaration module loader": + 'const [...holder] = [module]; holder[0].require("../shared/window.cjs");', + "object assignment module loader": + 'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require("../shared/window.cjs");', + "array assignment module loader": + 'const holder = {}; [holder.loader] = [module]; holder.loader.require("../shared/window.cjs");', + "nested assignment module loader": + 'const holder = {}; ({ inner: { loader: holder.loader } } = { inner: { loader: module } }); holder.loader.require("../shared/window.cjs");', + "default assignment module loader": + 'const holder = {}; ({ loader: holder.loader = module } = {}); holder.loader.require("../shared/window.cjs");', + "rest array assignment module loader": + 'const holder = {}; let ignored; [ignored, ...holder.loaders] = [{}, module]; holder.loaders[0].require("../shared/window.cjs");', + "delegate assignment module loader": + 'const holder = {}; ({ load: holder.load } = { load: module.require }); holder.load.call(module, "../shared/window.cjs");', + "rewrite assignment module loader": + 'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require = replacement; require("../shared/window.cjs");', + "exports assignment local loader": + 'const holder = {}; ({ exports: holder.loader } = module); holder.loader.require("../shared/window.cjs");', + "rest assignment local loader": + 'const holder = {}; ({ ...holder.loader } = module); holder.loader.require("../shared/window.cjs");', + "excluded assignment local loader": + 'const holder = {}; let ignored; ({ loader: ignored, ...holder.copy } = { loader: module }); holder.copy.loader.require("../shared/window.cjs");', + "inactive default assignment local loader": + 'const holder = {}; ({ loader: holder.loader = module } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");', + "overwritten assignment local loader": + 'const holder = { loader: module }; ({ loader: holder.loader } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");', + "bounded alias selection": aliasSelectionBudgetSource(), + "captured module delegate": + 'const load = module.require; load.call(module, "../shared/window.cjs");', + "destructured module delegate": + 'const { require: load } = module; load.apply(module, ["../shared/window.cjs"]);', + "selected local delegate slots": + 'const table = { x: {}, y: {}, z: module }; const target = flag ? table.x : table.y; target.require = replacement; require("../shared/window.cjs");', + "local assigned loader": + 'const holder = {}; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");', + "overwritten local loader": + 'const holder = { loader: module }; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");', + "normalized absolute ESM": 'import "/discard%2F/../shared/window.cjs";', + "normalized absolute dynamic import": + 'import("/discard%5C/../shared/window.cjs");', + "rewritten CommonJS delegate": + 'module.require = replacement; require("../shared/window.cjs");', + "nested rewritten CommonJS delegate": + 'const holder = { loader: module }; holder.loader.require = replacement; require("../shared/window.cjs");', + "assigned rewritten CommonJS delegate": + 'const holder = {}; holder.loader = module; holder.loader.require = replacement; require("../shared/window.cjs");', + "assigned module loader": + 'const holder = {}; holder.loader = module; holder.loader.require("../shared/window.cjs");', + "nested module loader": + 'const holder = { loader: module }; holder.loader.require("../shared/window.cjs");', + "destructured module loader": + 'const holder = { loader: module }; const { loader } = holder; loader.require("../shared/window.cjs");', + "array module loader": + 'const holder = [module]; holder[0].require("../shared/window.cjs");', + "bound module loader": + 'const load = module.require.bind(module); load("../shared/window.cjs");', + "bound module alias loader": + 'const current = module; const load = current.require.bind(current); load("../shared/window.cjs");', + "computed bound module loader": + 'const load = module["require"].bind(module); load("../shared/window.cjs");', + "fallback module alias": + 'const { loader = module } = options; loader.require("../shared/window.cjs");', + "inactive fallback": + 'const { loader = module } = { loader: {} }; loader.require("../shared/window.cjs");', + "absolute ESM": 'import "/shared/window.cjs";', + "invalid absolute ESM": 'import "/shared%2Fwindow.cjs";', + "invalid absolute dynamic import": 'import("/shared%5Cwindow.cjs");', + "absolute dynamic import": 'import("/shared/window.cjs");', + "absolute file URL": 'import "file:///shared/window.cjs";', + "dynamic file URL": 'import("file:///shared/window.cjs");', + "createRequire base": + 'import { createRequire } from "node:module"; const require = createRequire(new URL("../other/nested/main.cjs", import.meta.url)); require("../shared/window.cjs");', +}; + +const uncheckedModuleLoaders = new Set([ + "custom iterator assignment module loader", + "replaced iterator assignment module loader", + "custom iterator rest declaration module loader", + "logical rewrite assignment module loader", + "object rest declaration module loader", + "array rest declaration module loader", + "object assignment module loader", + "array assignment module loader", + "nested assignment module loader", + "default assignment module loader", + "rest array assignment module loader", + "delegate assignment module loader", + "rewrite assignment module loader", + "bounded alias selection", + "captured module delegate", + "destructured module delegate", + "rewritten CommonJS delegate", + "nested rewritten CommonJS delegate", + "assigned rewritten CommonJS delegate", + "assigned module loader", + "module.require", + "computed module.require", + "module alias", + "later reassigned module alias", + "conditional module alias", + "fallback module alias", + "bound module loader", + "bound module alias loader", + "computed bound module loader", + "nested module loader", + "destructured module loader", + "array module loader", +]); + +const fixtureMainPath = (loader: string): string => { + switch (loader) { + case "absolute ESM": + case "normalized absolute ESM": + case "normalized absolute dynamic import": + case "invalid absolute ESM": + case "invalid absolute dynamic import": + case "absolute dynamic import": + case "absolute file URL": + case "dynamic file URL": + case "createRequire base": + return "main.mjs"; + case "absolute main": + return "/opt/app/main.cjs"; + case "Windows relative main": + return "sub\\main.cjs"; + case "Windows drive-relative main": + return "C:main.cjs"; + case "percent main": + case "single percent main": + return "main%2Fentry.cjs"; + default: + return "main.cjs"; + } +}; + +const fixtureMainInventoryPath = (loader: string, main: string): string => { + if (loader === "absolute main") return "opt/app/main.cjs"; + if (loader === "Windows relative main") return "b/sub/main.cjs"; + if (loader === "Windows drive-relative main") return "b/main.cjs"; + return `b/${main}`; +}; + +const fixtureLoaderSource = (loader: string): string => + unprovenLoaderSources[loader] ?? + (loader === "resolve-only" + ? 'require.resolve("../shared/window.cjs");' + : loader === "missing wildcard alias" + ? 'require("#missing/*");' + : loader === "absolute require" + ? 'require("/shared/window.cjs");' + : loader === "projected module" + ? 'const { exports: api } = module; api.require = () => {}; api.require("../shared/window.cjs");' + : loader === "absolute main" + ? 'require("../../shared/window.cjs");' + : loader === "later reassigned module alias" + ? 'let current = module; current.require("../shared/window.cjs"); current = other;' + : loader === "earlier reassigned module alias" + ? 'let current = module; current = { require() {} }; current.require("../shared/window.cjs");' + : loader === "conditional module alias" + ? 'let current = module; if (flag) current = other; current.require("../shared/window.cjs");' + : loader === "module.require" + ? 'module.require("../shared/window.cjs");' + : loader === "computed module.require" + ? 'module["require"]("../shared/window.cjs");' + : loader === "module alias" + ? 'const current = module; current.require("../shared/window.cjs");' + : loader === "local module" + ? 'function local(module) { module.require("../shared/window.cjs"); }' + : loader === "pattern alias" + ? 'require("#window/window");' + : loader.endsWith("alias") + ? 'require("#entry");' + : 'require("../shared/window.cjs");'); + +const explicitCompetingLoaders = [ + "module.require", + "computed module.require", + "module alias", + "later reassigned module alias", + "earlier reassigned module alias", + "conditional module alias", + "absolute main", + "Windows relative main", + "Windows drive-relative main", + "percent main", + "pattern alias", + "recursive alias", + "external-package alias", + "builtin alias", + "electron alias", + "electron main alias", + "blocked alias", + "local module", + "projected module", + "resolve-only", + "missing wildcard alias", + "absolute require", +] as const; + +/** Every competing-loader case, in the order the original matrix ran. */ +export const competingLoaderNames = [ + ...explicitCompetingLoaders, + ...Object.keys(unprovenLoaderSources), + "single percent main", +]; + +const groupSize = Math.ceil(competingLoaderNames.length / 3); +/** Three disjoint groups so Vitest can run the matrix on separate workers. */ +export const competingLoaderGroups = [ + competingLoaderNames.slice(0, groupSize), + competingLoaderNames.slice(groupSize, groupSize * 2), + competingLoaderNames.slice(groupSize * 2), +] as const; + +/** One public CLI or MCP observation for a competing loader fixture. */ +export const assertCompetingLoaderOwnership = async ( + surface: "CLI" | "stdio MCP", + loader: string, + cli: TestCli, +): Promise => { + const root = await createTestTempDirectory("rea-loader-owner-"); + const single = loader === "single percent main"; + const inert = [ + "native array assignment local loader", + "exports assignment local loader", + "rest assignment local loader", + "excluded assignment local loader", + "inactive default assignment local loader", + "overwritten assignment local loader", + "local assigned loader", + "overwritten local loader", + "invalid absolute ESM", + "invalid absolute dynamic import", + "builtin alias", + "electron alias", + "electron main alias", + "inactive fallback", + "blocked alias", + "local module", + "projected module", + "resolve-only", + "missing wildcard alias", + "earlier reassigned module alias", + ].includes(loader); + const absolute = loader === "absolute main"; + const main = fixtureMainPath(loader); + const imports = fixtureImports(loader); + const load = fixtureLoaderSource(loader); + await writeFixtureFiles(root, { + "a/package.json": JSON.stringify({ main: "main.cjs" }), + "a/main.cjs": + single || absolute + ? "module.exports = 1;" + : 'require("../shared/window.cjs");', + "b/package.json": JSON.stringify({ main, imports }), + [fixtureMainInventoryPath(loader, main)]: load, + "b/proxy/window.cjs": 'require("../../shared/window.cjs");', + "b/node_modules/fixture-window/package.json": JSON.stringify({ + main: "index.cjs", + }), + "b/node_modules/fixture-window/index.cjs": + 'require("../../../shared/window.cjs");', + "shared/window.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "a/index.html": "first application page", + "b/index.html": "second application page", + "shared/index.html": "source-relative decoy", + "index.html": "artifact-root decoy", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + if (loader.startsWith("Windows ")) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "loads", + properties: expect.objectContaining({ + declared_path: main, + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + if ( + ["absolute require", "absolute ESM", "absolute dynamic import"].includes( + loader, + ) + ) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: "/shared/window.cjs", + resolution_status: "unavailable", + resolved_path: null, + }), + }), + ]), + ); + if (loader === "createRequire base") + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + kind: "require", + specifier: null, + resolved_path: null, + }), + }), + ]), + ); + if (loader.endsWith("alias") && !inert && !loader.includes("module alias")) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + specifier: loader === "pattern alias" ? "#window/window" : "#entry", + resolution_status: "unavailable", + }), + }), + ]), + ); + if (uncheckedModuleLoaders.has(loader)) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + kind: "require", + specifier: null, + resolved_path: null, + }), + }), + ]), + ); + const roles = graph.nodes.filter(({ kind }) => kind === "electron-renderer"); + expect(roles).toHaveLength(1); + const role = roles[0]; + const target = single ? "b/index.html" : inert ? "a/index.html" : null; + expect(role?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + resolved_path: target, + resolution_status: target === null ? "unavailable" : "resolved", + }), + }), + ]), + ); + expect( + graph.edges + .filter( + ({ source_node_id, relation }) => + source_node_id === role?.node_id && relation === "maps_to", + ) + .map(({ properties }) => properties.resolved_path), + ).toEqual(target === null ? [] : [target]); +}; diff --git a/tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts b/tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts new file mode 100644 index 0000000000..fe51424ee7 --- /dev/null +++ b/tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from "vitest"; + +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + assertCompetingLoaderOwnership, + competingLoaderGroups, + competingLoaderNames, +} from "./javascriptLiteralLoaderOwnership.js"; + +const loaders = competingLoaderGroups[0]; + +it("partitions every competing loader across the three ownership files", () => { + const assigned = competingLoaderGroups.flat(); + expect(assigned).toEqual(competingLoaderNames); + expect(new Set(assigned).size).toBe(competingLoaderNames.length); + for (const group of competingLoaderGroups) + expect(group.length).toBeGreaterThan(0); +}); + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + loaders.map((loader) => ({ surface, loader })), + ), +)( + "preserves competing loader ownership for $loader through $surface", + async ({ surface, loader }, { cli }) => { + await assertCompetingLoaderOwnership(surface, loader, cli); + }, +); diff --git a/tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts b/tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts new file mode 100644 index 0000000000..2f503d5e2a --- /dev/null +++ b/tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts @@ -0,0 +1,18 @@ +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + assertCompetingLoaderOwnership, + competingLoaderGroups, +} from "./javascriptLiteralLoaderOwnership.js"; + +const loaders = competingLoaderGroups[2]; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + loaders.map((loader) => ({ surface, loader })), + ), +)( + "preserves competing loader ownership for $loader through $surface", + async ({ surface, loader }, { cli }) => { + await assertCompetingLoaderOwnership(surface, loader, cli); + }, +); diff --git a/tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts b/tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts new file mode 100644 index 0000000000..7ebdee179f --- /dev/null +++ b/tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts @@ -0,0 +1,18 @@ +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + assertCompetingLoaderOwnership, + competingLoaderGroups, +} from "./javascriptLiteralLoaderOwnership.js"; + +const loaders = competingLoaderGroups[1]; + +cliTest.for( + (["CLI", "stdio MCP"] as const).flatMap((surface) => + loaders.map((loader) => ({ surface, loader })), + ), +)( + "preserves competing loader ownership for $loader through $surface", + async ({ surface, loader }, { cli }) => { + await assertCompetingLoaderOwnership(surface, loader, cli); + }, +); diff --git a/tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts b/tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts index 31fb7ae6df..61cc229264 100644 --- a/tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts +++ b/tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts @@ -4,30 +4,10 @@ import { join } from "node:path"; import { createPackageWithOptions } from "@electron/asar"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { aliasSelectionBudgetSource } from "../../fixtures/javascriptAliasSelectionBudget.js"; import { cliTest } from "../../support/cli/cliFixture.js"; import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; -const fixtureImports = (loader: string) => { - switch (loader) { - case "pattern alias": - return { "#window/*": "./proxy/*.cjs" }; - case "recursive alias": - return { "#entry": "#window", "#window": "./proxy/window.cjs" }; - case "external-package alias": - return { "#entry": "fixture-window" }; - case "builtin alias": - return { "#entry": "fs" }; - case "electron alias": - return { "#entry": "electron" }; - case "electron main alias": - return { "#entry": "electron/main" }; - default: - return { "#entry": null }; - } -}; - cliTest.for( (["CLI", "stdio MCP"] as const).flatMap((surface) => [ @@ -96,371 +76,6 @@ cliTest.for( }, ); -const unprovenLoaderSources: Readonly> = { - "custom iterator assignment module loader": - 'const values = { *[Symbol.iterator]() { yield module; } }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");', - "replaced iterator assignment module loader": - 'const values = [{}]; values[Symbol.iterator] = function* () { yield module; }; const holder = {}; [holder.loader] = values; holder.loader.require("../shared/window.cjs");', - "custom iterator rest declaration module loader": - 'const values = { *[Symbol.iterator]() { yield module; } }; const [...holder] = values; holder[0].require("../shared/window.cjs");', - "native array assignment local loader": - 'const holder = { loader: module }; [holder.loader] = [{ require() {} }]; holder.loader.require("../shared/window.cjs");', - "logical rewrite assignment module loader": - 'const holder = {}; holder.loader ??= module; holder.loader.require = replacement; require("../shared/window.cjs");', - "object rest declaration module loader": - 'const { ...holder } = { loader: module }; holder.loader.require("../shared/window.cjs");', - "array rest declaration module loader": - 'const [...holder] = [module]; holder[0].require("../shared/window.cjs");', - "object assignment module loader": - 'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require("../shared/window.cjs");', - "array assignment module loader": - 'const holder = {}; [holder.loader] = [module]; holder.loader.require("../shared/window.cjs");', - "nested assignment module loader": - 'const holder = {}; ({ inner: { loader: holder.loader } } = { inner: { loader: module } }); holder.loader.require("../shared/window.cjs");', - "default assignment module loader": - 'const holder = {}; ({ loader: holder.loader = module } = {}); holder.loader.require("../shared/window.cjs");', - "rest array assignment module loader": - 'const holder = {}; let ignored; [ignored, ...holder.loaders] = [{}, module]; holder.loaders[0].require("../shared/window.cjs");', - "delegate assignment module loader": - 'const holder = {}; ({ load: holder.load } = { load: module.require }); holder.load.call(module, "../shared/window.cjs");', - "rewrite assignment module loader": - 'const holder = {}; ({ loader: holder.loader } = { loader: module }); holder.loader.require = replacement; require("../shared/window.cjs");', - "exports assignment local loader": - 'const holder = {}; ({ exports: holder.loader } = module); holder.loader.require("../shared/window.cjs");', - "rest assignment local loader": - 'const holder = {}; ({ ...holder.loader } = module); holder.loader.require("../shared/window.cjs");', - "excluded assignment local loader": - 'const holder = {}; let ignored; ({ loader: ignored, ...holder.copy } = { loader: module }); holder.copy.loader.require("../shared/window.cjs");', - "inactive default assignment local loader": - 'const holder = {}; ({ loader: holder.loader = module } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");', - "overwritten assignment local loader": - 'const holder = { loader: module }; ({ loader: holder.loader } = { loader: { require() {} } }); holder.loader.require("../shared/window.cjs");', - "bounded alias selection": aliasSelectionBudgetSource(), - "captured module delegate": - 'const load = module.require; load.call(module, "../shared/window.cjs");', - "destructured module delegate": - 'const { require: load } = module; load.apply(module, ["../shared/window.cjs"]);', - "selected local delegate slots": - 'const table = { x: {}, y: {}, z: module }; const target = flag ? table.x : table.y; target.require = replacement; require("../shared/window.cjs");', - "local assigned loader": - 'const holder = {}; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");', - "overwritten local loader": - 'const holder = { loader: module }; holder.loader = { require() {} }; holder.loader.require("../shared/window.cjs");', - "normalized absolute ESM": 'import "/discard%2F/../shared/window.cjs";', - "normalized absolute dynamic import": - 'import("/discard%5C/../shared/window.cjs");', - "rewritten CommonJS delegate": - 'module.require = replacement; require("../shared/window.cjs");', - "nested rewritten CommonJS delegate": - 'const holder = { loader: module }; holder.loader.require = replacement; require("../shared/window.cjs");', - "assigned rewritten CommonJS delegate": - 'const holder = {}; holder.loader = module; holder.loader.require = replacement; require("../shared/window.cjs");', - "assigned module loader": - 'const holder = {}; holder.loader = module; holder.loader.require("../shared/window.cjs");', - "nested module loader": - 'const holder = { loader: module }; holder.loader.require("../shared/window.cjs");', - "destructured module loader": - 'const holder = { loader: module }; const { loader } = holder; loader.require("../shared/window.cjs");', - "array module loader": - 'const holder = [module]; holder[0].require("../shared/window.cjs");', - "bound module loader": - 'const load = module.require.bind(module); load("../shared/window.cjs");', - "bound module alias loader": - 'const current = module; const load = current.require.bind(current); load("../shared/window.cjs");', - "computed bound module loader": - 'const load = module["require"].bind(module); load("../shared/window.cjs");', - "fallback module alias": - 'const { loader = module } = options; loader.require("../shared/window.cjs");', - "inactive fallback": - 'const { loader = module } = { loader: {} }; loader.require("../shared/window.cjs");', - "absolute ESM": 'import "/shared/window.cjs";', - "invalid absolute ESM": 'import "/shared%2Fwindow.cjs";', - "invalid absolute dynamic import": 'import("/shared%5Cwindow.cjs");', - "absolute dynamic import": 'import("/shared/window.cjs");', - "absolute file URL": 'import "file:///shared/window.cjs";', - "dynamic file URL": 'import("file:///shared/window.cjs");', - "createRequire base": - 'import { createRequire } from "node:module"; const require = createRequire(new URL("../other/nested/main.cjs", import.meta.url)); require("../shared/window.cjs");', -}; - -const uncheckedModuleLoaders = new Set([ - "custom iterator assignment module loader", - "replaced iterator assignment module loader", - "custom iterator rest declaration module loader", - "logical rewrite assignment module loader", - "object rest declaration module loader", - "array rest declaration module loader", - "object assignment module loader", - "array assignment module loader", - "nested assignment module loader", - "default assignment module loader", - "rest array assignment module loader", - "delegate assignment module loader", - "rewrite assignment module loader", - "bounded alias selection", - "captured module delegate", - "destructured module delegate", - "rewritten CommonJS delegate", - "nested rewritten CommonJS delegate", - "assigned rewritten CommonJS delegate", - "assigned module loader", - "module.require", - "computed module.require", - "module alias", - "later reassigned module alias", - "conditional module alias", - "fallback module alias", - "bound module loader", - "bound module alias loader", - "computed bound module loader", - "nested module loader", - "destructured module loader", - "array module loader", -]); - -const fixtureMainPath = (loader: string): string => { - switch (loader) { - case "absolute ESM": - case "normalized absolute ESM": - case "normalized absolute dynamic import": - case "invalid absolute ESM": - case "invalid absolute dynamic import": - case "absolute dynamic import": - case "absolute file URL": - case "dynamic file URL": - case "createRequire base": - return "main.mjs"; - case "absolute main": - return "/opt/app/main.cjs"; - case "Windows relative main": - return "sub\\main.cjs"; - case "Windows drive-relative main": - return "C:main.cjs"; - case "percent main": - case "single percent main": - return "main%2Fentry.cjs"; - default: - return "main.cjs"; - } -}; - -const fixtureMainInventoryPath = (loader: string, main: string): string => { - if (loader === "absolute main") return "opt/app/main.cjs"; - if (loader === "Windows relative main") return "b/sub/main.cjs"; - if (loader === "Windows drive-relative main") return "b/main.cjs"; - return `b/${main}`; -}; - -const fixtureLoaderSource = (loader: string): string => - unprovenLoaderSources[loader] ?? - (loader === "resolve-only" - ? 'require.resolve("../shared/window.cjs");' - : loader === "missing wildcard alias" - ? 'require("#missing/*");' - : loader === "absolute require" - ? 'require("/shared/window.cjs");' - : loader === "projected module" - ? 'const { exports: api } = module; api.require = () => {}; api.require("../shared/window.cjs");' - : loader === "absolute main" - ? 'require("../../shared/window.cjs");' - : loader === "later reassigned module alias" - ? 'let current = module; current.require("../shared/window.cjs"); current = other;' - : loader === "earlier reassigned module alias" - ? 'let current = module; current = { require() {} }; current.require("../shared/window.cjs");' - : loader === "conditional module alias" - ? 'let current = module; if (flag) current = other; current.require("../shared/window.cjs");' - : loader === "module.require" - ? 'module.require("../shared/window.cjs");' - : loader === "computed module.require" - ? 'module["require"]("../shared/window.cjs");' - : loader === "module alias" - ? 'const current = module; current.require("../shared/window.cjs");' - : loader === "local module" - ? 'function local(module) { module.require("../shared/window.cjs"); }' - : loader === "pattern alias" - ? 'require("#window/window");' - : loader.endsWith("alias") - ? 'require("#entry");' - : 'require("../shared/window.cjs");'); - -cliTest.for( - (["CLI", "stdio MCP"] as const).flatMap((surface) => - [ - "module.require", - "computed module.require", - "module alias", - "later reassigned module alias", - "earlier reassigned module alias", - "conditional module alias", - "absolute main", - "Windows relative main", - "Windows drive-relative main", - "percent main", - "pattern alias", - "recursive alias", - "external-package alias", - "builtin alias", - "electron alias", - "electron main alias", - "blocked alias", - "local module", - "projected module", - "resolve-only", - "missing wildcard alias", - "absolute require", - ...Object.keys(unprovenLoaderSources), - "single percent main", - ].map((loader) => ({ surface, loader })), - ), -)( - "preserves competing loader ownership for $loader through $surface", - async ({ surface, loader }, { cli }) => { - const root = await createTestTempDirectory("rea-loader-owner-"); - const single = loader === "single percent main"; - const inert = [ - "native array assignment local loader", - "exports assignment local loader", - "rest assignment local loader", - "excluded assignment local loader", - "inactive default assignment local loader", - "overwritten assignment local loader", - "local assigned loader", - "overwritten local loader", - "invalid absolute ESM", - "invalid absolute dynamic import", - "builtin alias", - "electron alias", - "electron main alias", - "inactive fallback", - "blocked alias", - "local module", - "projected module", - "resolve-only", - "missing wildcard alias", - "earlier reassigned module alias", - ].includes(loader); - const absolute = loader === "absolute main"; - const main = fixtureMainPath(loader); - const imports = fixtureImports(loader); - const load = fixtureLoaderSource(loader); - await writeFixtureFiles(root, { - "a/package.json": JSON.stringify({ main: "main.cjs" }), - "a/main.cjs": - single || absolute - ? "module.exports = 1;" - : 'require("../shared/window.cjs");', - "b/package.json": JSON.stringify({ main, imports }), - [fixtureMainInventoryPath(loader, main)]: load, - "b/proxy/window.cjs": 'require("../../shared/window.cjs");', - "b/node_modules/fixture-window/package.json": JSON.stringify({ - main: "index.cjs", - }), - "b/node_modules/fixture-window/index.cjs": - 'require("../../../shared/window.cjs");', - "shared/window.cjs": - 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', - "a/index.html": "first application page", - "b/index.html": "second application page", - "shared/index.html": "source-relative decoy", - "index.html": "artifact-root decoy", - }); - const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); - if (loader.startsWith("Windows ")) - expect(graph.edges).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - relation: "loads", - properties: expect.objectContaining({ - declared_path: main, - resolution_status: "unavailable", - resolved_path: null, - }), - }), - ]), - ); - if ( - ["absolute require", "absolute ESM", "absolute dynamic import"].includes( - loader, - ) - ) - expect(graph.edges).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - relation: "imports", - properties: expect.objectContaining({ - specifier: "/shared/window.cjs", - resolution_status: "unavailable", - resolved_path: null, - }), - }), - ]), - ); - if (loader === "createRequire base") - expect(graph.edges).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - relation: "imports", - properties: expect.objectContaining({ - kind: "require", - specifier: null, - resolved_path: null, - }), - }), - ]), - ); - if (loader.endsWith("alias") && !inert && !loader.includes("module alias")) - expect(graph.edges).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - relation: "imports", - properties: expect.objectContaining({ - specifier: - loader === "pattern alias" ? "#window/window" : "#entry", - resolution_status: "unavailable", - }), - }), - ]), - ); - if (uncheckedModuleLoaders.has(loader)) - expect(graph.edges).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - relation: "imports", - properties: expect.objectContaining({ - kind: "require", - specifier: null, - resolved_path: null, - }), - }), - ]), - ); - const roles = graph.nodes.filter( - ({ kind }) => kind === "electron-renderer", - ); - expect(roles).toHaveLength(1); - const role = roles[0]; - const target = single ? "b/index.html" : inert ? "a/index.html" : null; - expect(role?.observations).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - properties: expect.objectContaining({ - resolved_path: target, - resolution_status: target === null ? "unavailable" : "resolved", - }), - }), - ]), - ); - expect( - graph.edges - .filter( - ({ source_node_id, relation }) => - source_node_id === role?.node_id && relation === "maps_to", - ) - .map(({ properties }) => properties.resolved_path), - ).toEqual(target === null ? [] : [target]); - }, -); - cliTest.for(["CLI", "stdio MCP"] as const)( "propagates the application owner into an inventoried ASAR through %s", async (surface, { cli }) => { diff --git a/tests/boundary/filesystem/ciTestSequencer.test.ts b/tests/boundary/filesystem/ciTestSequencer.test.ts index 320bef0b22..dc0945779a 100644 --- a/tests/boundary/filesystem/ciTestSequencer.test.ts +++ b/tests/boundary/filesystem/ciTestSequencer.test.ts @@ -8,10 +8,10 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { allocateByDuration, - estimatedShardCost, parseTestDurations, readTestDurations, - weightFileDurations, + scheduleWall, + seedSplitLoaderDurations, } from "../../ciTestSequencer.js"; describe("CI test history", () => { @@ -86,28 +86,43 @@ describe("CI test history", () => { }); }); +describe("split loader history", () => { + const source = + "tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts"; + const parts = [ + "tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts", + "tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts", + "tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts", + ]; + + it("divides a pre-split duration across the four current files", () => { + const seeded = seedSplitLoaderDurations( + new Map([ + [source, 192_000], + ["tests/other.test.ts", 10], + ]), + ); + expect(seeded.get(source)).toBe(54_000); + for (const path of parts) expect(seeded.get(path)).toBe(46_000); + expect(seeded.get("tests/other.test.ts")).toBe(10); + expect( + [...seeded.values()].reduce((sum, duration) => sum + duration, 0), + ).toBe(192_000 + 10); + }); + + it("keeps published durations once any split file is in history", () => { + const recorded = new Map([ + [source, 20_000], + [parts[0]!, 30_000], + ]); + expect(seedSplitLoaderDurations(recorded)).toEqual(recorded); + expect( + seedSplitLoaderDurations(new Map([["tests/other.test.ts", 10]])), + ).toEqual(new Map([["tests/other.test.ts", 10]])); + }); +}); + describe("duration shard allocation", () => { - it("charges serial process-boundary files their full duration and other projects one worker share", () => { - expect(estimatedShardCost("process-boundary", 400)).toBe(400); - expect(estimatedShardCost("domain", 400)).toBe(200); - expect(estimatedShardCost("adapters", 400)).toBe(200); - }); - - it("fills unknown files from the raw median before serial and shared weighting", () => { - expect( - weightFileDurations([ - { project: "domain", duration: 400 }, - { project: "process-boundary", duration: undefined }, - { project: "adapters", duration: undefined }, - ]), - ).toEqual([200, 400, 200]); - expect( - weightFileDurations([ - { project: "process-boundary", duration: undefined }, - ]), - ).toEqual([undefined]); - }); - it.prop([ fc.uniqueArray(fc.integer({ min: 0, max: 10_000 }), { minLength: 1, @@ -188,4 +203,72 @@ describe("duration shard allocation", () => { expect(shards!.flat()).toHaveLength(items.length); expect(new Set(shards!.flat()).size).toBe(items.length); }); + + it("keeps each file whole and runs the serial group after the parallel group", () => { + const parallel = (id: string, duration: number) => ({ + item: id, + key: id, + duration, + laneWidth: 2, + groupOrder: 0, + }); + const serial = (id: string, duration: number) => ({ + item: id, + key: id, + duration, + laneWidth: 1, + groupOrder: 1, + }); + const paired = [ + parallel("wide", 300), + parallel("peer", 300), + parallel("third", 300), + ]; + expect(scheduleWall(paired.slice(0, 1), 1)).toBe(300); + expect(scheduleWall(paired.slice(0, 2), 1)).toBe(300); + expect(scheduleWall(paired, 1)).toBe(600); + expect( + scheduleWall([parallel("ordinary", 100), serial("tail", 60)], 1), + ).toBe(160); + expect(() => + scheduleWall( + [parallel("wide", 10), { ...parallel("wide", 10), laneWidth: 1 }], + 1, + ), + ).toThrow(RangeError); + + const packed = [ + parallel("ordinary", 100), + serial("tail-a", 60), + serial("tail-b", 60), + ]; + expect(allocateByDuration(packed, 2)).toEqual([ + ["ordinary"], + ["tail-a", "tail-b"], + ]); + const spread = [ + parallel("wide", 300), + serial("tail-a", 100), + serial("tail-b", 100), + parallel("small", 40), + ]; + const shards = allocateByDuration(spread, 2); + expect(shards).toEqual([["wide"], ["tail-a", "tail-b", "small"]]); + expect( + scheduleWall( + spread.filter((entry) => entry.item === "wide"), + 1, + ), + ).toBe(300); + expect( + Math.max( + ...shards!.map((ids) => + scheduleWall( + spread.filter((entry) => ids.includes(entry.item)), + 1, + ), + ), + ), + ).toBe(300); + }); }); diff --git a/tests/ciTestSequencer.ts b/tests/ciTestSequencer.ts index 49de240f5b..cb56f53d01 100644 --- a/tests/ciTestSequencer.ts +++ b/tests/ciTestSequencer.ts @@ -60,32 +60,138 @@ export async function readTestDurations( } } -type TimedItem = { item: T; key: string; duration: number | undefined }; +/** + * Case counts from the competing-loader file before it was split: 54 cases + * stayed in the original file and 46 moved into each of the three new files. + * History that still names only the original path is that combined cost. + */ +const LOADER_HISTORY_SOURCE = + "tests/acceptance/applications/javascriptLiteralPercentPaths.test.ts"; +const LOADER_HISTORY_PARTS = [ + "tests/acceptance/applications/javascriptLiteralLoaderOwnershipEarly.test.ts", + "tests/acceptance/applications/javascriptLiteralLoaderOwnershipMiddle.test.ts", + "tests/acceptance/applications/javascriptLiteralLoaderOwnershipLate.test.ts", +] as const; +const LOADER_HISTORY_REMAINDER_CASES = 54; +const LOADER_HISTORY_PART_CASES = 46; +const LOADER_HISTORY_CASES = + LOADER_HISTORY_REMAINDER_CASES + + LOADER_HISTORY_PART_CASES * LOADER_HISTORY_PARTS.length; -// process-boundary runs its files one at a time. Other projects overlap up to -// the runner worker cap, so equal file-duration sums are not equal wall time. -const PARALLEL_TEST_WORKERS = 2; - -export function estimatedShardCost( - project: string, - durationMs: number, -): number { - if (project === "process-boundary") return durationMs; - return durationMs / PARALLEL_TEST_WORKERS; +/** + * Give the split loader files a share of the pre-split duration. Once history + * records any of the new paths, their published durations stand. + */ +export function seedSplitLoaderDurations( + durations: ReadonlyMap, +): Map { + const combined = durations.get(LOADER_HISTORY_SOURCE); + if ( + combined === undefined || + LOADER_HISTORY_PARTS.some((path) => durations.has(path)) + ) + return new Map(durations); + const seeded = new Map(durations); + seeded.set( + LOADER_HISTORY_SOURCE, + (combined * LOADER_HISTORY_REMAINDER_CASES) / LOADER_HISTORY_CASES, + ); + for (const path of LOADER_HISTORY_PARTS) + seeded.set( + path, + (combined * LOADER_HISTORY_PART_CASES) / LOADER_HISTORY_CASES, + ); + return seeded; } -/** Weight raw durations, filling gaps from the raw median before worker sharing. */ -export function weightFileDurations( - entries: readonly { project: string; duration: number | undefined }[], -): (number | undefined)[] { - const known = entries - .flatMap(({ duration }) => (duration === undefined ? [] : [duration])) - .sort((left, right) => left - right); - if (!known.length) return entries.map(() => undefined); - const lower = known[Math.floor((known.length - 1) / 2)]!; - const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2; - return entries.map(({ project, duration }) => - estimatedShardCost(project, duration ?? median), +type TimedItem = { + item: T; + key: string; + duration: number | undefined; + /** + * Workers for this file's sequence group. The file still occupies one worker + * for its full duration; the other lanes only overlap its peers. + */ + laneWidth?: number; + /** + * Vitest sequence group. Groups run from lowest to highest, so a later group + * adds its own makespan instead of overlapping an earlier one. + */ + groupOrder?: number; +}; + +type Phase = { laneWidth: number; lanes: number[] }; + +const laneWidthOf = (entry: TimedItem): number => { + const laneWidth = entry.laneWidth ?? 1; + if (!Number.isInteger(laneWidth) || laneWidth < 1) + throw new RangeError("laneWidth must be a positive integer"); + return laneWidth; +}; + +const groupOrderOf = (entry: TimedItem): number => { + const groupOrder = entry.groupOrder ?? 0; + if (!Number.isInteger(groupOrder) || groupOrder < 0) + throw new RangeError("groupOrder must be a non-negative integer"); + return groupOrder; +}; + +const durationOf = (entry: TimedItem, unknownMs: number): number => { + const duration = entry.duration ?? unknownMs; + if (!Number.isFinite(duration) || duration <= 0) + throw new RangeError("duration must be positive"); + return duration; +}; + +const wallOf = (phases: ReadonlyMap): number => { + let wall = 0; + for (const phase of phases.values()) + wall += phase.lanes.reduce((max, lane) => Math.max(max, lane), 0); + return wall; +}; + +/** Place one whole file on the least-loaded lane of its sequence group. */ +const place = ( + phases: ReadonlyMap, + entry: TimedItem, + duration: number, +): Map => { + const groupOrder = groupOrderOf(entry); + const laneWidth = laneWidthOf(entry); + const existing = phases.get(groupOrder); + if (existing && existing.laneWidth !== laneWidth) + throw new RangeError("one sequence group must keep a single lane width"); + const lanes = existing + ? existing.lanes.slice() + : Array.from({ length: laneWidth }, () => 0); + let shortest = 0; + for (let index = 1; index < lanes.length; index += 1) + if (lanes[index]! < lanes[shortest]!) shortest = index; + lanes[shortest] = lanes[shortest]! + duration; + const next = new Map(phases); + next.set(groupOrder, { laneWidth, lanes }); + return next; +}; + +/** + * Wall time of one shard. Each file keeps its full duration on one worker. + * Files overlap only inside a sequence group, up to that group's lane width, + * and later groups run after earlier ones. + */ +export function scheduleWall( + items: readonly TimedItem[], + unknownMs: number, +): number { + const ordered = items.toSorted( + (left, right) => + durationOf(right, unknownMs) - durationOf(left, unknownMs) || + (left.key < right.key ? -1 : left.key > right.key ? 1 : 0), + ); + return wallOf( + ordered.reduce( + (phases, entry) => place(phases, entry, durationOf(entry, unknownMs)), + new Map(), + ), ); } @@ -104,19 +210,29 @@ export function allocateByDuration( const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2; const ordered = items.toSorted( (left, right) => - (right.duration ?? median) - (left.duration ?? median) || + durationOf(right, median) - durationOf(left, median) || (left.key < right.key ? -1 : left.key > right.key ? 1 : 0), ); const shards = Array.from({ length: shardCount }, () => ({ - duration: 0, + phases: new Map(), items: [] as T[], })); for (const entry of ordered) { - const lightest = shards.reduce((left, right) => - right.duration < left.duration ? right : left, - ); - lightest.items.push(entry.item); - lightest.duration += entry.duration ?? median; + const duration = durationOf(entry, median); + let chosen = shards[0]!; + let chosenPhases = place(chosen.phases, entry, duration); + let chosenWall = wallOf(chosenPhases); + for (const candidate of shards.slice(1)) { + const phases = place(candidate.phases, entry, duration); + const wall = wallOf(phases); + if (wall < chosenWall) { + chosen = candidate; + chosenPhases = phases; + chosenWall = wall; + } + } + chosen.phases = chosenPhases; + chosen.items.push(entry.item); } return shards.map(({ items }) => items); } @@ -129,26 +245,29 @@ export class CiTestSequencer extends BaseSequencer { const shard = this.ctx.config.shard; if (!historyPath || !shard) return super.shard(files); const root = this.ctx.config.root; - const durations = await readTestDurations(historyPath, root); - if (!durations) return super.shard(files); + const recorded = await readTestDurations(historyPath, root); + if (!recorded) return super.shard(files); + const durations = seedSplitLoaderDurations(recorded); - const timed = files.map((item) => { + const weighted = files.map((item) => { const path = relative(root, resolve(root, item.moduleId)) .split(sep) .join("/"); + const projectWorkers = item.project.config.maxWorkers; + const laneWidth = + typeof projectWorkers === "number" && + Number.isInteger(projectWorkers) && + projectWorkers >= 1 + ? projectWorkers + : 1; return { item, key: path + "\0" + item.project.name + "\0" + item.pool, - project: item.project.name, duration: durations.get(path), + laneWidth, + groupOrder: item.project.config.sequence.groupOrder, }; }); - const costs = weightFileDurations(timed); - const weighted = timed.map((entry, index) => ({ - item: entry.item, - key: entry.key, - duration: costs[index], - })); const shards = allocateByDuration(weighted, shard.count); return shards ? shards[shard.index - 1]! : super.shard(files); } From d23be5c0898c39c9f0c54e2e0b43ee746d92d316 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:56:35 +0800 Subject: [PATCH 08/35] fix(web): preserve static WebMCP schema property coverage Reuse the shared object-member owner for literal spreads, retain known property names under unresolved membership, and distinguish complete, partial, unknown, and absent projections. Keep historical coverage omission unchanged and preserve own keys across prototype setters. Refs #5075 --- docs/browser-observation.md | 16 + .../javascript/javascriptAstValues.test.ts | 186 +++++++++ src/domain/javascript/javascriptAstValues.ts | 165 ++++++-- src/domain/webBundleAnalysis.ts | 12 +- src/domain/webBundleAnalyzer.test.ts | 373 +++++++++++++++++- src/domain/webBundleAnalyzerAst.ts | 20 +- src/domain/webBundleAnalyzerInspection.ts | 67 +++- .../webBundleSchemaCoverageCliMcp.test.ts | 133 +++++++ 8 files changed, 884 insertions(+), 88 deletions(-) create mode 100644 tests/acceptance/browser/webBundleSchemaCoverageCliMcp.test.ts diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 64823a4728..6034644493 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -19,8 +19,24 @@ Inspect native SAZ or HAR 1.2 offline with REA, or connect Everywhere's separate - `analyze_web_bundle` / `rea analyze-web-bundle` parses selected script artifacts without execution and derives chunk edges, route and endpoint candidates, vendor fingerprints, static WebMCP declarations, and optionally source-map/original-source evidence. - `observe_web_session` / `rea observe-web-session` arms an observation window of the requested duration for an external user action and records ordered reload, SPA navigation, redirect, failure, lifecycle, and target-termination metadata. - `discover_webmcp_tools` / `rea discover-webmcp-tools` uses the experimental CDP WebMCP domain to return every in-scope registration and its complete observed `input_schema` declaration and canonical `input_schema_sha256` digest inline. Declarations are untrusted JSON data, including declared types, required fields, enums, examples and reference URLs. REA does not resolve schema references. These fields replace `input_schema_shape`. Registrations with the same name and URL in distinct observed frames retain separate identities. `tool_key` includes the observed frame ID; it is not a stable identity across navigation or browser sessions. Same-frame registration updates replace that owner only, and removal leaves other owners intact. Declared `readOnly`, `untrustedContent`, `consequential`, and `autosubmit` annotation hints are reported per registration: `true`/`false` are declared values, `null` means the declaration did not provide the hint in that capture, and a missing `consequential` key means the hint was not recorded, either in a historical capture or because the observed annotation was invalid. Consult discovery completeness and exclusions to distinguish these cases. Malformed annotation values retain the registration with incomplete coverage and an `invalid_protocol_value` exclusion. Other malformed protocol payloads can fail discovery explicitly. REA never exposes `WebMCP.invokeTool`. + - `compare_web_captures` / `rea compare-web-captures` compares passive captures or exact-step-aligned browser scenarios. Scenario results commit reproducible literal normalization and expose alignment failures plus artifact-level action, screenshot, DOM, accessibility, URL, history, storage, and event differences. Missing or truncated evidence is never treated as equivalence. +For `analyze_web_bundle`, each `webmcp_declarations` entry's +`schema_property_names` is a source-derived projection of statically known own +property keys, not schema validation. `schema_property_names_coverage` reports +how complete that projection is: `complete` means a literal properties object +was fully enumerated (including an empty literal); `partial` means a literal +object was available but some computed or spread membership was unresolved, +and the name array retains keys that were known; `unknown` means the schema or +properties owner could not be enumerated as an object literal, including +explicit non-object values; `not_present` means no effective schema or own +`properties` member was present after the `inputSchema ?? input_schema` alias +selection. `not_present` describes this projection only: it does not validate a +schema or rule out properties expressed through other schema constructs. Older +observations may omit the coverage field; treat omission as `unknown` rather +than inferring completeness from the array. + Passive DOM comparison checks complete node inventories and complete frame parent/child structure independently. Frame IDs and sibling enumeration order are capture-local and do not establish a change; duplicate IDs, missing parents, diff --git a/src/domain/javascript/javascriptAstValues.test.ts b/src/domain/javascript/javascriptAstValues.test.ts index 457cbc3258..67324136fe 100644 --- a/src/domain/javascript/javascriptAstValues.test.ts +++ b/src/domain/javascript/javascriptAstValues.test.ts @@ -4,6 +4,7 @@ import { expect, it } from "vitest"; import { semanticStaticPropertyKey, staticObjectProperty, + staticObjectPropertyNames, } from "./javascriptAstValues.js"; import { SEMANTIC_EXPRESSION_DEPTH_LIMIT } from "./javascriptSemanticResourceLimits.js"; @@ -70,3 +71,188 @@ it("bounds nested inline object spreads without losing a shallow final property" .status, ).toBe("dynamic"); }); + +it("collects complete own-key sets through bounded literal spreads", () => { + const names = staticObjectPropertyNames( + t.objectExpression([ + t.spreadElement( + t.objectExpression([ + t.objectProperty(t.identifier("query"), t.objectExpression([])), + t.objectProperty(t.identifier("shared"), t.numericLiteral(1)), + ]), + ), + t.objectProperty(t.identifier("direct"), t.objectExpression([])), + t.objectProperty(t.identifier("shared"), t.numericLiteral(2)), + ]), + ); + expect(names).toEqual({ + status: "complete", + names: ["query", "shared", "direct"], + }); +}); + +it("keeps key enumeration unknown when object membership is unresolved", () => { + const dynamicSpread = t.identifier("dynamicProperties"); + const computedKey = t.identifier("propertyName"); + const cases = [ + t.objectExpression([ + t.spreadElement(dynamicSpread), + t.objectProperty(t.identifier("direct"), t.objectExpression([])), + ]), + t.objectExpression([ + t.objectProperty(computedKey, t.objectExpression([]), true), + t.objectProperty(t.identifier("direct"), t.objectExpression([])), + ]), + t.objectExpression([ + t.spreadElement( + t.objectExpression([ + t.objectMethod( + "get", + t.identifier("query"), + [], + t.blockStatement([]), + ), + ]), + ), + ]), + ]; + for (const object of cases) + expect(staticObjectPropertyNames(object).status).toBe("partial"); +}); + +it("retains known keys alongside unresolved object membership", () => { + expect( + staticObjectPropertyNames( + t.objectExpression([ + t.spreadElement(t.identifier("dynamicProperties")), + t.objectProperty(t.identifier("direct"), t.objectExpression([])), + t.objectProperty( + t.identifier("propertyName"), + t.objectExpression([]), + true, + ), + ]), + ), + ).toEqual({ status: "partial", names: ["direct"] }); +}); + +it("does not claim a complete key set after exceeding spread depth", () => { + let object = t.objectExpression([ + t.objectProperty(t.identifier("query"), t.objectExpression([])), + ]); + for (let depth = 0; depth < SEMANTIC_EXPRESSION_DEPTH_LIMIT; depth++) + object = t.objectExpression([t.spreadElement(object)]); + expect(staticObjectPropertyNames(object).status).toBe("complete"); + expect( + staticObjectPropertyNames(t.objectExpression([t.spreadElement(object)])) + .status, + ).toBe("partial"); +}); + +it("enumerates a wide inline spread without a variadic call overflow", () => { + const properties = Array.from({ length: 130_000 }, (_, index) => + t.objectProperty(t.identifier(`key${index}`), t.numericLiteral(index)), + ); + const result = staticObjectPropertyNames( + t.objectExpression([t.spreadElement(t.objectExpression(properties))]), + ); + expect(result.status).toBe("complete"); + if (result.status === "complete") { + expect(result.names).toHaveLength(properties.length); + expect(result.names[0]).toBe("key0"); + expect(result.names.at(-1)).toBe("key129999"); + } +}); + +it("excludes prototype setters from own keys but preserves other __proto__ forms", () => { + const setter = t.objectProperty( + t.identifier("__proto__"), + t.objectExpression([ + t.objectProperty(t.identifier("inherited"), t.numericLiteral(1)), + ]), + ); + expect( + staticObjectPropertyNames( + t.objectExpression([ + setter, + t.objectProperty(t.identifier("own"), t.numericLiteral(2)), + ]), + ), + ).toEqual({ status: "complete", names: ["own"] }); + + const ownForms = [ + t.objectProperty(t.stringLiteral("__proto__"), t.numericLiteral(1), true), + t.objectProperty( + t.identifier("__proto__"), + t.identifier("__proto__"), + false, + true, + ), + t.objectMethod( + "method", + t.identifier("__proto__"), + [], + t.blockStatement([]), + ), + ]; + for (const member of ownForms) + expect(staticObjectPropertyNames(t.objectExpression([member]))).toEqual({ + status: "complete", + names: ["__proto__"], + }); +}); + +it("keeps inherited lookup uncertain after a top-level prototype setter", () => { + const setter = t.objectProperty( + t.identifier("__proto__"), + t.objectExpression([ + t.objectProperty(t.identifier("inherited"), t.numericLiteral(1)), + ]), + ); + expect( + staticObjectProperty(t.objectExpression([setter]), "inherited").status, + ).toBe("dynamic"); + expect( + staticObjectProperty( + t.objectExpression([ + setter, + t.objectProperty(t.identifier("inherited"), t.numericLiteral(2)), + ]), + "inherited", + ).status, + ).toBe("explicit"); + expect( + staticObjectProperty( + t.objectExpression([ + t.objectProperty(t.identifier("inherited"), t.numericLiteral(2)), + setter, + ]), + "inherited", + ).status, + ).toBe("explicit"); + const spreadSource = t.objectExpression([ + setter, + t.objectProperty(t.identifier("visible"), t.numericLiteral(3)), + ]); + expect( + staticObjectProperty( + t.objectExpression([t.spreadElement(spreadSource)]), + "inherited", + ).status, + ).toBe("missing"); +}); + +it("omits prototype setters copied by literal object spread", () => { + const source = t.objectExpression([ + t.objectProperty( + t.identifier("__proto__"), + t.objectExpression([ + t.objectProperty(t.identifier("hidden"), t.numericLiteral(1)), + ]), + ), + t.objectProperty(t.identifier("visible"), t.numericLiteral(2)), + ]); + expect( + staticObjectPropertyNames(t.objectExpression([t.spreadElement(source)])), + ).toEqual({ status: "complete", names: ["visible"] }); +}); diff --git a/src/domain/javascript/javascriptAstValues.ts b/src/domain/javascript/javascriptAstValues.ts index f0b971100d..d4b50588b0 100644 --- a/src/domain/javascript/javascriptAstValues.ts +++ b/src/domain/javascript/javascriptAstValues.ts @@ -130,6 +130,79 @@ type StaticObjectProperty = | { readonly status: "missing" | "dynamic" } | { readonly status: "explicit"; readonly property: t.ObjectProperty }; +type StaticObjectMember = + | { readonly kind: "unknown" } + | { readonly kind: "prototype-setter" } + | { + readonly kind: "property"; + readonly property: t.ObjectProperty | t.ObjectMethod; + }; + +const UNKNOWN_STATIC_OBJECT_MEMBER: StaticObjectMember = { kind: "unknown" }; + +const isObjectPrototypeSetter = (property: t.ObjectProperty): boolean => + !property.computed && + !property.shorthand && + semanticStaticPropertyKey(property.key, false) === "__proto__"; + +/** Expand only bounded literal spreads, preserving unknown writes in order. */ +const appendStaticObjectMembers = ( + members: StaticObjectMember[], + object: t.ObjectExpression, + depth: number, + isNullish?: (node: t.Node) => boolean, +): void => { + for (const property of object.properties) { + if (!t.isSpreadElement(property)) { + if (t.isObjectProperty(property) && isObjectPrototypeSetter(property)) { + // A top-level setter changes lookup inheritance but is never an own + // key; one copied through object spread has no effect on the target. + if (depth === 0) members.push({ kind: "prototype-setter" }); + continue; + } + members.push({ kind: "property", property }); + continue; + } + const { node: spread, depth: wrappers } = unwrapJavaScriptExpression( + property.argument, + ); + if (t.isObjectExpression(spread)) { + const nextDepth = depth + wrappers + 1; + // Copying a getter can run code that changes another source key. + if ( + nextDepth > SEMANTIC_EXPRESSION_DEPTH_LIMIT || + spread.properties.some((entry) => + t.isObjectMethod(entry, { kind: "get" }), + ) + ) { + members.push(UNKNOWN_STATIC_OBJECT_MEMBER); + continue; + } + appendStaticObjectMembers(members, spread, nextDepth, isNullish); + continue; + } + // These primitive boxes have no own keys; strings can own index keys. + if ( + t.isBooleanLiteral(spread) || + t.isNumericLiteral(spread) || + t.isBigIntLiteral(spread) || + isNullish?.(property.argument) === true + ) + continue; + members.push(UNKNOWN_STATIC_OBJECT_MEMBER); + } +}; + +const staticObjectMembers = ( + object: t.ObjectExpression, + depth: number, + isNullish?: (node: t.Node) => boolean, +): readonly StaticObjectMember[] => { + const members: StaticObjectMember[] = []; + appendStaticObjectMembers(members, object, depth, isNullish); + return members; +}; + /** Find the effective explicit property, preserving unresolved overrides. */ export const staticObjectProperty = ( object: t.Node | undefined, @@ -138,49 +211,57 @@ export const staticObjectProperty = ( ): StaticObjectProperty => { if (object === undefined) return { status: "missing" }; if (!t.isObjectExpression(object)) return { status: "dynamic" }; - const lookup = ( - current: t.ObjectExpression, - depth: number, - ): StaticObjectProperty => { - for (const property of current.properties.toReversed()) { - if (t.isSpreadElement(property)) { - const { node: spread, depth: wrappers } = unwrapJavaScriptExpression( - property.argument, - ); - if (t.isObjectExpression(spread)) { - const nextDepth = depth + wrappers + 1; - // Copying a getter can run code that changes another source key. - if ( - nextDepth > SEMANTIC_EXPRESSION_DEPTH_LIMIT || - spread.properties.some((entry) => - t.isObjectMethod(entry, { kind: "get" }), - ) - ) - return { status: "dynamic" }; - const found = lookup(spread, nextDepth); - if (found.status === "missing") continue; - return found; - } - // These primitive boxes have no own keys; strings can own index keys. - if ( - t.isBooleanLiteral(spread) || - t.isNumericLiteral(spread) || - t.isBigIntLiteral(spread) || - isNullish?.(property.argument) === true - ) - continue; - return { status: "dynamic" }; - } - const key = semanticStaticPropertyKey(property.key, property.computed); - if (key === null) return { status: "dynamic" }; - if (key !== name) continue; - return t.isObjectProperty(property) - ? { status: "explicit", property } - : { status: "dynamic" }; + let prototypeMaySupply = false; + for (const member of staticObjectMembers(object, 0, isNullish).toReversed()) { + if (member.kind === "prototype-setter") { + prototypeMaySupply = true; + continue; } - return { status: "missing" }; - }; - return lookup(object, 0); + if (member.kind === "unknown") return { status: "dynamic" }; + const { property } = member; + const key = semanticStaticPropertyKey(property.key, property.computed); + if (key === null) return { status: "dynamic" }; + if (key !== name) continue; + return t.isObjectProperty(property) + ? { status: "explicit", property } + : { status: "dynamic" }; + } + return prototypeMaySupply ? { status: "dynamic" } : { status: "missing" }; +}; + +export type StaticObjectPropertyNames = + | { readonly status: "missing" | "dynamic" } + | { readonly status: "partial"; readonly names: readonly string[] } + | { readonly status: "complete"; readonly names: readonly string[] }; + +/** Retain known own names and report unresolved object membership separately. */ +export const staticObjectPropertyNames = ( + object: t.Node | undefined, + isNullish?: (node: t.Node) => boolean, +): StaticObjectPropertyNames => { + if (object === undefined) return { status: "missing" }; + if (!t.isObjectExpression(object)) return { status: "dynamic" }; + const names = new Set(); + let complete = true; + for (const member of staticObjectMembers(object, 0, isNullish)) { + if (member.kind === "prototype-setter") continue; + if (member.kind === "unknown") { + complete = false; + continue; + } + const key = semanticStaticPropertyKey( + member.property.key, + member.property.computed, + ); + if (key === null) { + complete = false; + continue; + } + names.add(key); + } + return complete + ? { status: "complete", names: [...names] } + : { status: "partial", names: [...names] }; }; /** Later properties own a key; an intervening dynamic write leaves it unknown. */ diff --git a/src/domain/webBundleAnalysis.ts b/src/domain/webBundleAnalysis.ts index 4eef3cf949..d25625b92e 100644 --- a/src/domain/webBundleAnalysis.ts +++ b/src/domain/webBundleAnalysis.ts @@ -220,7 +220,17 @@ export const webBundleAnalysisSchema = z.object({ z.object({ name: z.string().nullable(), description: z.string().nullable(), - schema_property_names: z.array(z.string()), + schema_property_names: z + .array(z.string()) + .describe( + "Statically known own property names from the declared schema projection; a partial list may omit unresolved members.", + ), + schema_property_names_coverage: z + .enum(["complete", "partial", "unknown", "not_present"]) + .optional() + .describe( + "Coverage of the schema property-name projection. Missing means unknown for historical observations. Not a schema-validity result.", + ), trust: z.literal("page-declared-untrusted"), location: sourceLocationSchema, }), diff --git a/src/domain/webBundleAnalyzer.test.ts b/src/domain/webBundleAnalyzer.test.ts index c1ad169ff7..f7af053d30 100644 --- a/src/domain/webBundleAnalyzer.test.ts +++ b/src/domain/webBundleAnalyzer.test.ts @@ -158,6 +158,232 @@ it("discloses widened returned-reference effects only for the affected script", ).toEqual(["/candidate"]); }); +it.each([ + { + label: "inline declaration and schema spreads", + source: `document.modelContext.registerTool({ + ...{ + name: "spread-tool", + description: "Copied from a literal", + inputSchema: { + ...{ + type: "object", + properties: { query: { type: "string" } } + } + } + } + });`, + name: "spread-tool", + description: "Copied from a literal", + properties: ["query"], + coverage: "complete", + }, + { + label: "unrelated trailing literal spread", + source: `document.modelContext.registerTool({ + name: "direct-tool", + description: "Explicit metadata", + inputSchema: { properties: { query: { type: "string" } } }, + ...{ timeout: 5 } + });`, + name: "direct-tool", + description: "Explicit metadata", + properties: ["query"], + coverage: "complete", + }, + { + label: "null spread after explicit metadata", + source: `document.modelContext.registerTool({ + name: "null-safe-tool", + description: "Null spread is inert", + inputSchema: { properties: { query: { type: "string" } } }, + ...null + });`, + name: "null-safe-tool", + description: "Null spread is inert", + properties: ["query"], + coverage: "complete", + }, + { + label: "snake-case schema property from a spread", + source: `document.modelContext.registerTool({ + ...{ + name: "snake-tool", + description: "Snake-case schema", + input_schema: { properties: { item_id: { type: "string" } } } + } + });`, + name: "snake-tool", + description: "Snake-case schema", + properties: ["item_id"], + coverage: "complete", + }, +])( + "preserves WebMCP facts through $label", + ({ source, name, description, properties, coverage }) => { + const result = analyzeCapturedWebBundle(inspection(source)); + + expect(result.observations.webmcp_declarations).toEqual([ + expect.objectContaining({ + name, + description, + schema_property_names: properties, + schema_property_names_coverage: coverage, + trust: "page-declared-untrusted", + }), + ]); + expect(result.completeness.status).toBe("complete"); + }, +); + +it.each([ + { + label: "unknown spread after explicit facts", + source: `document.modelContext.registerTool({ + name: "possibly-overridden", + description: "Possibly overridden", + inputSchema: { properties: { query: { type: "string" } } }, + ...dynamicOptions + });`, + name: null, + description: null, + properties: [], + coverage: "unknown", + }, + { + label: "explicit facts after an unknown spread", + source: `document.modelContext.registerTool({ + ...dynamicOptions, + name: "known-after-spread", + description: "Explicitly restored", + inputSchema: { properties: { query: { type: "string" } } } + });`, + name: "known-after-spread", + description: "Explicitly restored", + properties: ["query"], + coverage: "complete", + }, +])( + "preserves rightmost uncertainty for $label", + ({ source, name, description, properties, coverage }) => { + const result = analyzeCapturedWebBundle(inspection(source)); + + expect(result.observations.webmcp_declarations).toEqual([ + expect.objectContaining({ + name, + description, + schema_property_names: properties, + schema_property_names_coverage: coverage, + trust: "page-declared-untrusted", + }), + ]); + expect(result.completeness.status).toBe("complete"); + }, +); + +it.each([ + { + label: "an unknown camel-case schema override", + source: `document.modelContext.registerTool({ + ...dynamicOptions, + inputSchema: { properties: { camel: {} } }, + ...anotherDynamicOptions, + input_schema: { properties: { snake: {} } }, + name: "tool", + description: "Explicit metadata" + });`, + name: "tool", + properties: [], + coverage: "unknown", + }, + { + label: "an explicit null camel-case schema", + source: `document.modelContext.registerTool({ + name: "null-alias", + description: "Fallback applies", + inputSchema: null, + input_schema: { properties: { snake: {} } } + });`, + name: "null-alias", + properties: ["snake"], + coverage: "complete", + }, + { + label: "literal property spreads", + source: `document.modelContext.registerTool({ + name: "spread-properties", + inputSchema: { properties: { ...{ query: {} }, direct: {} } } + });`, + name: "spread-properties", + properties: ["direct", "query"], + coverage: "complete", + }, + { + label: "prototype setters are not schema properties", + source: `document.modelContext.registerTool({ + name: "prototype-properties", + inputSchema: { properties: { __proto__: { hidden: {} }, direct: {}, ["__proto__"]: {} } } + });`, + name: "prototype-properties", + properties: ["__proto__", "direct"], + coverage: "complete", + }, + { + label: "own schema properties before a prototype setter", + source: `document.modelContext.registerTool({ + name: "own-before-prototype", + inputSchema: { properties: { known: {} }, __proto__: { properties: { inherited: {} } } } + });`, + name: "own-before-prototype", + properties: ["known"], + coverage: "complete", + }, + { + label: "own schema properties after a prototype setter", + source: `document.modelContext.registerTool({ + name: "own-after-prototype", + inputSchema: { __proto__: { properties: { inherited: {} } }, properties: { known: {} } } + });`, + name: "own-after-prototype", + properties: ["known"], + coverage: "complete", + }, + { + label: "an unresolved property spread", + source: `document.modelContext.registerTool({ + name: "unknown-properties", + inputSchema: { properties: { ...dynamicProperties, direct: {} } } + });`, + name: "unknown-properties", + properties: ["direct"], + coverage: "partial", + }, + { + label: "an unresolved computed property key", + source: `document.modelContext.registerTool({ + name: "computed-properties", + inputSchema: { properties: { [propertyName]: {}, direct: {} } } + });`, + name: "computed-properties", + properties: ["direct"], + coverage: "partial", + }, +])( + "preserves WebMCP schema ownership for $label", + ({ source, name, properties, coverage }) => { + const result = analyzeCapturedWebBundle(inspection(source)); + + expect(result.observations.webmcp_declarations).toEqual([ + expect.objectContaining({ + name, + schema_property_names: properties, + schema_property_names_coverage: coverage, + trust: "page-declared-untrusted", + }), + ]); + }, +); + describe("web bundle analyzer", () => { it("extracts graph, route, endpoint, vendor, and WebMCP evidence", () => { const source = ` @@ -206,6 +432,7 @@ describe("web bundle analyzer", () => { name: "lookup-user", trust: "page-declared-untrusted", schema_property_names: ["userId", "verbose"], + schema_property_names_coverage: "complete", }), ]); expect(result.inferences).toEqual( @@ -266,6 +493,7 @@ describe("web bundle analyzer", () => { name, description, schema_property_names: [propertyName], + schema_property_names_coverage: "complete", trust: "page-declared-untrusted", }); }); @@ -1481,10 +1709,24 @@ describe("web metadata key semantics", () => { expect.objectContaining({ name: "new", schema_property_names: ["literal"], + schema_property_names_coverage: "partial", + }), + expect.objectContaining({ + name: null, + schema_property_names_coverage: "unknown", + }), + expect.objectContaining({ + name: null, + schema_property_names_coverage: "not_present", + }), + expect.objectContaining({ + name: "restored", + schema_property_names_coverage: "unknown", + }), + expect.objectContaining({ + name: "valid", + schema_property_names_coverage: "not_present", }), - expect.objectContaining({ name: null }), - expect.objectContaining({ name: "restored" }), - expect.objectContaining({ name: "valid" }), ]); }); @@ -1514,20 +1756,86 @@ describe("web metadata key semantics", () => { expect( result.observations.webmcp_declarations.map( - ({ name, schema_property_names }) => ({ name, schema_property_names }), + ({ name, schema_property_names, schema_property_names_coverage }) => ({ + name, + schema_property_names, + schema_property_names_coverage, + }), ), ).toEqual([ - { name: "literal", schema_property_names: ["", "properties", "type"] }, - { name: "missing properties", schema_property_names: [] }, - { name: "dynamic properties", schema_property_names: [] }, - { name: "null properties", schema_property_names: [] }, - { name: "missing schema", schema_property_names: [] }, - { name: "dynamic schema", schema_property_names: [] }, - { name: "null schema", schema_property_names: [] }, - { name: "spread schema", schema_property_names: [] }, - { name: "computed literal keys", schema_property_names: ["", "literal"] }, + { + name: "literal", + schema_property_names: ["", "properties", "type"], + schema_property_names_coverage: "partial", + }, + { + name: "missing properties", + schema_property_names: [], + schema_property_names_coverage: "not_present", + }, + { + name: "dynamic properties", + schema_property_names: [], + schema_property_names_coverage: "unknown", + }, + { + name: "null properties", + schema_property_names: [], + schema_property_names_coverage: "unknown", + }, + { + name: "missing schema", + schema_property_names: [], + schema_property_names_coverage: "not_present", + }, + { + name: "dynamic schema", + schema_property_names: [], + schema_property_names_coverage: "unknown", + }, + { + name: "null schema", + schema_property_names: [], + schema_property_names_coverage: "not_present", + }, + { + name: "spread schema", + schema_property_names: [], + schema_property_names_coverage: "unknown", + }, + { + name: "computed literal keys", + schema_property_names: ["", "literal"], + schema_property_names_coverage: "partial", + }, ]); }); + + it("keeps missing coverage absent when parsing historical observations", () => { + const analyzed = analyzeCapturedWebBundle( + inspection( + `document.modelContext.registerTool({ name: "legacy", inputSchema: { properties: {} } });`, + ), + ); + const declaration = analyzed.observations.webmcp_declarations[0]; + expect(declaration).toBeDefined(); + if (declaration === undefined) throw new Error("Expected declaration"); + const { schema_property_names_coverage: _coverage, ...legacyDeclaration } = + declaration; + const parsed = webBundleAnalysisSchema.parse({ + ...analyzed, + observations: { + ...analyzed.observations, + webmcp_declarations: [legacyDeclaration], + }, + }); + expect( + Object.hasOwn( + parsed.observations.webmcp_declarations[0] ?? {}, + "schema_property_names_coverage", + ), + ).toBe(false); + }); }); it("retains WebMCP declarations through optional member and call chaining", () => { @@ -1695,6 +2003,8 @@ it("retains distinct WebMCP descriptions and names while deduplicating equivalen document.modelContext.registerTool({ name: "lookup", description: "second", inputSchema: { properties: { alpha: {} } } }); document.modelContext.registerTool({ name: "equivalent", inputSchema: { properties: { alpha: {}, beta: {} } } }); document.modelContext.registerTool({ name: "equivalent", inputSchema: { properties: { beta: {}, alpha: {}, alpha: {} } } }); + document.modelContext.registerTool({ name: "empty-status", inputSchema: { properties: {} } }); + document.modelContext.registerTool({ name: "empty-status", inputSchema: { properties: dynamicProperties } }); document.modelContext.registerTool({ description: "unnamed" }); document.modelContext.registerTool({ name: "", description: "unnamed" }); `), @@ -1702,10 +2012,16 @@ it("retains distinct WebMCP descriptions and names while deduplicating equivalen expect( result.observations.webmcp_declarations.map( - ({ name, description, schema_property_names }) => ({ + ({ name, description, schema_property_names, + schema_property_names_coverage, + }) => ({ + name, + description, + schema_property_names, + schema_property_names_coverage, }), ), ).toEqual([ @@ -1713,19 +2029,44 @@ it("retains distinct WebMCP descriptions and names while deduplicating equivalen name: "lookup", description: "first", schema_property_names: ["alpha"], + schema_property_names_coverage: "complete", }, { name: "lookup", description: "second", schema_property_names: ["alpha"], + schema_property_names_coverage: "complete", }, { name: "equivalent", description: null, schema_property_names: ["alpha", "beta"], + schema_property_names_coverage: "complete", + }, + { + name: "empty-status", + description: null, + schema_property_names: [], + schema_property_names_coverage: "complete", + }, + { + name: "empty-status", + description: null, + schema_property_names: [], + schema_property_names_coverage: "unknown", + }, + { + name: null, + description: "unnamed", + schema_property_names: [], + schema_property_names_coverage: "not_present", + }, + { + name: "", + description: "unnamed", + schema_property_names: [], + schema_property_names_coverage: "not_present", }, - { name: null, description: "unnamed", schema_property_names: [] }, - { name: "", description: "unnamed", schema_property_names: [] }, ]); expect(result.observations.webmcp_declarations[2]?.location).toEqual({ script_key: `scr_${"1".repeat(64)}`, diff --git a/src/domain/webBundleAnalyzerAst.ts b/src/domain/webBundleAnalyzerAst.ts index ce98e1b959..f9972a27f5 100644 --- a/src/domain/webBundleAnalyzerAst.ts +++ b/src/domain/webBundleAnalyzerAst.ts @@ -2,28 +2,18 @@ import * as t from "@babel/types"; import { sanitizeBrowserUrl } from "./browserObservation.js"; import { + isNullishJavaScriptSyntax, readExactJavaScriptLiteral, - semanticStaticPropertyName, + staticObjectProperty, } from "./javascript/javascriptAstValues.js"; -/** Pick the endpoint argument for common network-call patterns. */ - +/** Read a statically known own property value from an object expression. */ export const objectValue = ( object: t.ObjectExpression, name: string, ): t.ObjectProperty["value"] | undefined => { - let value: t.ObjectProperty["value"] | undefined; - for (const property of object.properties) { - if (t.isSpreadElement(property)) { - value = undefined; - continue; - } - const key = semanticStaticPropertyName(property.key, property.computed); - if (key === "" && !t.isStringLiteral(property.key)) value = undefined; - else if (key === name) - value = t.isObjectProperty(property) ? property.value : undefined; - } - return value; + const result = staticObjectProperty(object, name, isNullishJavaScriptSyntax); + return result.status === "explicit" ? result.property.value : undefined; }; export const objectString = ( diff --git a/src/domain/webBundleAnalyzerInspection.ts b/src/domain/webBundleAnalyzerInspection.ts index 696a049f8f..a9a36b78c0 100644 --- a/src/domain/webBundleAnalyzerInspection.ts +++ b/src/domain/webBundleAnalyzerInspection.ts @@ -12,6 +12,9 @@ import { isIntrinsicMapLookup } from "./javascript/javascriptSemanticMapOperatio import { semanticStaticPropertyKey, semanticStaticPropertyName, + isNullishJavaScriptSyntax, + staticObjectProperty, + staticObjectPropertyNames, unwrapJavaScriptExpression, } from "./javascript/javascriptAstValues.js"; import { @@ -23,7 +26,6 @@ import { location, locationFields, objectString, - objectValue, resolveSpecifier, isUrlLikeModuleSpecifier, } from "./webBundleAnalyzerAst.js"; @@ -365,23 +367,60 @@ const addWebMcpDeclaration = ( if (!t.isObjectExpression(declaration)) return; const name = objectString(declaration, "name"); const description = objectString(declaration, "description"); - const schema = - objectValue(declaration, "inputSchema") ?? - objectValue(declaration, "input_schema"); - const properties = t.isObjectExpression(schema) - ? objectValue(schema, "properties") - : undefined; - const schemaPropertyNames = t.isObjectExpression(properties) - ? properties.properties.flatMap((property) => { - if (!t.isObjectProperty(property)) return []; - const key = semanticStaticPropertyKey(property.key, property.computed); - return key === null ? [] : [key]; - }) - : []; + const camelSchema = staticObjectProperty( + declaration, + "inputSchema", + isNullishJavaScriptSyntax, + ); + const schemaProperty = + camelSchema.status === "missing" || + (camelSchema.status === "explicit" && + isNullishJavaScriptSyntax(camelSchema.property.value)) + ? staticObjectProperty( + declaration, + "input_schema", + isNullishJavaScriptSyntax, + ) + : camelSchema; + let schemaPropertyNames: readonly string[] = []; + let schemaPropertyNamesCoverage: NonNullable< + WebMcpDeclaration["schema_property_names_coverage"] + > = "unknown"; + if (schemaProperty.status === "missing") { + schemaPropertyNamesCoverage = "not_present"; + } else if (schemaProperty.status === "explicit") { + const schema = schemaProperty.property.value; + if (t.isObjectExpression(schema)) { + const propertiesProperty = staticObjectProperty( + schema, + "properties", + isNullishJavaScriptSyntax, + ); + if (propertiesProperty.status === "missing") { + schemaPropertyNamesCoverage = "not_present"; + } else if (propertiesProperty.status === "explicit") { + const properties = propertiesProperty.property.value; + if (t.isObjectExpression(properties)) { + const projection = staticObjectPropertyNames( + properties, + isNullishJavaScriptSyntax, + ); + if ( + projection.status === "complete" || + projection.status === "partial" + ) { + schemaPropertyNames = projection.names; + schemaPropertyNamesCoverage = projection.status; + } + } + } + } + } const metadata = { name: name ?? null, description: description ?? null, schema_property_names: [...new Set(schemaPropertyNames)].sort(), + schema_property_names_coverage: schemaPropertyNamesCoverage, }; // Deduplicate the metadata we return, preserving omission and descriptions. // Property names are a variable-length source-derived list: NUL-joins diff --git a/tests/acceptance/browser/webBundleSchemaCoverageCliMcp.test.ts b/tests/acceptance/browser/webBundleSchemaCoverageCliMcp.test.ts new file mode 100644 index 0000000000..657fa19f15 --- /dev/null +++ b/tests/acceptance/browser/webBundleSchemaCoverageCliMcp.test.ts @@ -0,0 +1,133 @@ +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const capturedSource = ` +const dynamic = globalThis.options; +document.modelContext.registerTool({ + name: "spread", + inputSchema: { properties: { ...{ query: {} }, direct: {} } } +}); +document.modelContext.registerTool({ + name: "partial", + inputSchema: { properties: { known: {}, ...dynamic } } +}); +document.modelContext.registerTool({ + name: "empty-status", description: "same", + inputSchema: { properties: dynamic } +}); +document.modelContext.registerTool({ + name: "empty-status", description: "same", + inputSchema: { properties: {} } +}); +document.modelContext.registerTool({ + name: "own-after-prototype-setter", + inputSchema: { + properties: { own: {} }, + __proto__: { properties: { inherited: {} } } + } +}); +`; + +const inspectDeclarations = (document: unknown) => { + const parsed = + toolContract("analyze_web_bundle").outputSchema.parse(document); + const parsedEvidence = parseEvidence(parsed); + return { + evidence: parsedEvidence, + declarations: parsed.normalized_result.observations.webmcp_declarations, + }; +}; + +const expectCoverageCases = ( + declarations: ReturnType["declarations"], +) => { + expect(declarations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "spread", + schema_property_names: ["direct", "query"], + schema_property_names_coverage: "complete", + }), + expect.objectContaining({ + name: "partial", + schema_property_names: ["known"], + schema_property_names_coverage: "partial", + }), + expect.objectContaining({ + name: "empty-status", + description: "same", + schema_property_names: [], + schema_property_names_coverage: "unknown", + }), + expect.objectContaining({ + name: "empty-status", + description: "same", + schema_property_names: [], + schema_property_names_coverage: "complete", + }), + expect.objectContaining({ + name: "own-after-prototype-setter", + schema_property_names: ["own"], + schema_property_names_coverage: "complete", + }), + ]), + ); + expect( + declarations.filter(({ name }) => name === "empty-status"), + ).toHaveLength(2); +}; + +cliTest( + "preserves WebMCP schema-name coverage through compiled CLI and stdio MCP", + async ({ cli }) => { + const browser = await startFakeCdpBrowser({ + commandResult: ({ method }) => + method === "Debugger.getScriptSource" + ? { scriptSource: capturedSource } + : undefined, + }); + try { + const contract = toolContract("analyze_web_bundle"); + const cliReply = await cli.run({ + arguments: [ + "analyze-web-bundle", + browser.endpoint, + "allowed-page", + "--allowed-origins", + browser.allowedOrigin, + "--observation-ms", + "0", + "--json", + ], + }); + expect(cliReply.exitCode, cliReply.stdout + cliReply.stderr).toBe(0); + const cliAnalysis = inspectDeclarations(cliReply.json); + expectCoverageCases(cliAnalysis.declarations); + + const { call } = await connectLocalToolsMcp(); + const input = { + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + include_script_sources: true, + fetch_source_maps: false, + }; + const response = await call(contract.name, input); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcpAnalysis = inspectDeclarations(response.structuredContent); + expectCoverageCases(mcpAnalysis.declarations); + expect(mcpAnalysis.evidence.operation).toBe("analyze_web_bundle"); + expect(mcpAnalysis.evidence.authority).toBe("external-service"); + expect(mcpAnalysis.declarations).toEqual(cliAnalysis.declarations); + } finally { + await browser.close(); + } + }, + 120_000, +); From 6b101ff23b4151806d3241cfa7cd0a2457eed566 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:16:44 +0800 Subject: [PATCH 09/35] fix(errors): preserve caller abort reasons at admission (#5671) * fix(errors): preserve caller abort reasons at admission Keep the opaque caller reason on reverse, pcap, and SQLite cancellation, and leave a recognized physical digest or command failure in place when a late abort is also observed. Fixes #5670 Co-authored-by: morluto * fix(reverse): keep a foreign digest ABORT_ERR as an adapter failure Treat ABORT_ERR as cancellation on the command boundary, where that code already meant an aborted provider process. A digest or stat rejection with the same code stays a physical adapter failure unless the caller signal is aborted. Cover aborted command EACCES, ETIMEDOUT, and ABORT_ERR with and without captured output. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- .../pcap/PcapAnalysisService.test.ts | 28 +++ src/application/pcap/PcapAnalysisService.ts | 6 +- .../reverse/ReverseEngineeringService.ts | 68 +++++- .../sqlite/SqliteDatabaseService.test.ts | 22 ++ .../sqlite/SqliteDatabaseService.ts | 6 +- .../reverseEngineeringAdmission.test.ts | 204 ++++++++++++++++++ 6 files changed, 324 insertions(+), 10 deletions(-) create mode 100644 src/application/pcap/PcapAnalysisService.test.ts diff --git a/src/application/pcap/PcapAnalysisService.test.ts b/src/application/pcap/PcapAnalysisService.test.ts new file mode 100644 index 0000000000..ec7e64763f --- /dev/null +++ b/src/application/pcap/PcapAnalysisService.test.ts @@ -0,0 +1,28 @@ +import { expect, it } from "vitest"; + +import { PcapAnalysisService } from "./PcapAnalysisService.js"; + +it("preserves the exact caller reason when pcap admission is already aborted", async () => { + const reason = Object.freeze({ requested: "stop" }); + const controller = new AbortController(); + controller.abort(reason); + let calls = 0; + const service = new PcapAnalysisService({ + inspectAvailability: () => Promise.reject(new Error("unused")), + close: () => Promise.resolve(), + execute: () => { + calls += 1; + return Promise.reject(new Error("provider must not run")); + }, + }); + const result = await service.execute( + "inspect_pcap_client", + {}, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(reason); + expect(calls).toBe(0); +}); diff --git a/src/application/pcap/PcapAnalysisService.ts b/src/application/pcap/PcapAnalysisService.ts index 23b968e582..cf2c8a94c3 100644 --- a/src/application/pcap/PcapAnalysisService.ts +++ b/src/application/pcap/PcapAnalysisService.ts @@ -24,7 +24,11 @@ export class PcapAnalysisService { options?: ExecutionOptions, ): Promise> { if (options?.signal?.aborted === true) - return err(new AnalysisCancelledError(operation)); + return err( + new AnalysisCancelledError(operation, { + cause: options.signal.reason, + }), + ); const request = pcapRequestSchema.safeParse({ operation, input }); if (!request.success) return err(new AnalysisInputError(operation, { cause: request.error })); diff --git a/src/application/reverse/ReverseEngineeringService.ts b/src/application/reverse/ReverseEngineeringService.ts index 0979313365..b082ac9d14 100644 --- a/src/application/reverse/ReverseEngineeringService.ts +++ b/src/application/reverse/ReverseEngineeringService.ts @@ -250,7 +250,9 @@ export class ReverseEngineeringService { signal?: AbortSignal, ): Promise> { if (signal?.aborted) - return Promise.resolve(err(new AnalysisCancelledError(operation))); + return Promise.resolve( + err(new AnalysisCancelledError(operation, { cause: signal.reason })), + ); if (this.#closing) return Promise.resolve( err( @@ -311,7 +313,10 @@ export class ReverseEngineeringService { signal?: AbortSignal, ): Promise> { const { executable: command, arguments: args } = profile; - if (signal?.aborted) return err(new AnalysisCancelledError(operation)); + if (signal?.aborted) + return err( + new AnalysisCancelledError(operation, { cause: signal.reason }), + ); const cleanup = await this.retryCleanup(operation); if (!cleanup.ok) return cleanup; const absolutePath = @@ -322,8 +327,12 @@ export class ReverseEngineeringService { digest = await this.#digest(absolutePath, signal, this.#stat); } catch (cause) { const code = commandFailureCode(cause); - if (signal?.aborted) - return err(new AnalysisCancelledError(operation, { cause })); + if (preflightCancelled(signal, cause, code)) + return err( + new AnalysisCancelledError(operation, { + cause: callerCancellationCause(signal, cause), + }), + ); if (code === "EACCES" || code === "EPERM") { const deniedPath = absolutePath === undefined ? command : absolutePath; return err( @@ -464,9 +473,9 @@ export class ReverseEngineeringService { code === "ENOENT" || code === "ERR_PROVIDER_LAUNCH") ) { - if (signal?.aborted || code === "ABORT_ERR") + if (commandCancelled(signal, cause, code)) earlyFailure = new AnalysisCancelledError(operation, { - cause, + cause: callerCancellationCause(signal, cause), ...ownedCleanupOptions(cause), }); else if (code === "ETIMEDOUT") @@ -520,9 +529,9 @@ export class ReverseEngineeringService { : null, signal: capture.signal, }; - if (signal?.aborted || code === "ABORT_ERR") + if (commandCancelled(signal, cause, code)) failure = new AnalysisCancelledError(operation, { - cause, + cause: callerCancellationCause(signal, cause), capturedOutput, ...ownedCleanupOptions(cause), }); @@ -605,6 +614,7 @@ export class ReverseEngineeringService { } if (signal?.aborted && failure === undefined) { failure = new AnalysisCancelledError(operation, { + cause: signal.reason, capturedOutput: { stdout, stderr, @@ -972,6 +982,48 @@ const digestFile = async ( }); }; +const PHYSICAL_FAILURE_CODES = new Set([ + "EACCES", + "EPERM", + "ETIMEDOUT", + "ENOENT", + "ENOTDIR", + "EINVAL", + "ELOOP", + "ENAMETOOLONG", + "E2BIG", + "EBUSY", + "ERR_PROVIDER_LAUNCH", +]); + +/** + * A late abort does not replace a recognized physical failure. Other rejections + * observed while the caller signal is aborted remain cancellation. A bare + * `ABORT_ERR` is not cancellation until the command boundary, which already + * used that code for an aborted provider process. + */ +const preflightCancelled = ( + signal: AbortSignal | undefined, + cause: unknown, + code: string | number | undefined, +): boolean => { + if (cause instanceof AnalysisCancelledError) return true; + if (signal?.aborted !== true) return false; + if (cause === signal.reason) return true; + return typeof code !== "string" || !PHYSICAL_FAILURE_CODES.has(code); +}; + +const commandCancelled = ( + signal: AbortSignal | undefined, + cause: unknown, + code: string | number | undefined, +): boolean => code === "ABORT_ERR" || preflightCancelled(signal, cause, code); + +const callerCancellationCause = ( + signal: AbortSignal | undefined, + cause: unknown, +): unknown => (signal?.aborted === true ? signal.reason : cause); + const commandFailureCode = (cause: unknown): string | number | undefined => { if (typeof cause !== "object" || cause === null) return undefined; const code: unknown = Reflect.get(cause, "code"); diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts index 543cd0b027..4fbd6e5172 100644 --- a/src/application/sqlite/SqliteDatabaseService.test.ts +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -145,3 +145,25 @@ it("preserves completed observed evidence when cancellation arrives after provid value: { normalized_result: value }, }); }); + +it("preserves the exact caller reason when inspection is already aborted", async () => { + const reason = Object.freeze({ requested: "stop" }); + const controller = new AbortController(); + controller.abort(reason); + let calls = 0; + const service = new SqliteDatabaseService({ + inspect: () => { + calls += 1; + return Promise.reject(new Error("provider must not run")); + }, + }); + const result = await service.inspect( + { path: "/selected.db" }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(reason); + expect(calls).toBe(0); +}); diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts index 7f8aaeb73e..b6b47a1a2f 100644 --- a/src/application/sqlite/SqliteDatabaseService.ts +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -71,7 +71,11 @@ export class SqliteDatabaseService { options?: ExecutionOptions, ): Promise> { if (options?.signal?.aborted) - return err(new AnalysisCancelledError(OPERATION)); + return err( + new AnalysisCancelledError(OPERATION, { + cause: options.signal.reason, + }), + ); const input = inspectSqliteDatabaseInputSchema.safeParse(rawInput); if (!input.success) return err( diff --git a/tests/composition/reverseEngineeringAdmission.test.ts b/tests/composition/reverseEngineeringAdmission.test.ts index 1b92f587a1..8440827b34 100644 --- a/tests/composition/reverseEngineeringAdmission.test.ts +++ b/tests/composition/reverseEngineeringAdmission.test.ts @@ -101,3 +101,207 @@ it("bounds concurrent reverse-engineering provider calls", async () => { await rm(directory, { recursive: true, force: true }); } }); + +it("preserves the exact caller reason when objdump admission is already aborted", async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-reverse-preabort-")); + const path = join(directory, "fixture.bin"); + await writeFile(path, "fixture"); + const reason = Object.freeze({ requested: "stop" }); + const controller = new AbortController(); + controller.abort(reason); + let runs = 0; + const service = new TestReverseEngineeringService({ + run: async () => { + runs += 1; + return { stdout: "", stderr: "" }; + }, + }); + try { + const result = await service.inspectWithObjdump( + { path, operation: "symbols" }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(reason); + expect(runs).toBe(0); + } finally { + await service.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +it("preserves the caller reason when cancellation arrives after provider output", async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-reverse-postabort-")); + const path = join(directory, "fixture.bin"); + await writeFile(path, "fixture"); + const reason = Object.freeze({ requested: "stop-after-output" }); + const controller = new AbortController(); + const service = new TestReverseEngineeringService({ + run: async () => { + controller.abort(reason); + return { stdout: "symbols", stderr: "" }; + }, + }); + try { + const result = await service.inspectWithObjdump( + { path, operation: "symbols" }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(reason); + expect(result.error.partialObservation).toMatchObject({ + raw_result: { stdout: "symbols" }, + }); + } finally { + await service.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +it("keeps a physical digest denial primary when the caller aborts during that failure", async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-reverse-digest-deny-")); + const path = join(directory, "fixture.bin"); + await writeFile(path, "fixture"); + const reason = Object.freeze({ requested: "stop-during-denial" }); + const controller = new AbortController(); + const denied = Object.assign(new Error("permission denied"), { + code: "EACCES", + }); + let runs = 0; + const service = new TestReverseEngineeringService({ + digest: async () => { + controller.abort(reason); + throw denied; + }, + run: async () => { + runs += 1; + return { stdout: "", stderr: "" }; + }, + }); + try { + const result = await service.inspectWithObjdump( + { path, operation: "symbols" }, + { signal: controller.signal }, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisAccessDeniedError" }, + }); + if (result.ok) return; + expect(result.error.cause).toBe(denied); + expect(runs).toBe(0); + } finally { + await service.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +it("keeps an unrelated digest ABORT_ERR as an adapter failure", async () => { + const directory = await mkdtemp(join(tmpdir(), "rea-reverse-digest-abort-")); + const path = join(directory, "fixture.bin"); + await writeFile(path, "fixture"); + const denied = Object.assign(new Error("digest aborted elsewhere"), { + code: "ABORT_ERR", + }); + const service = new TestReverseEngineeringService({ + digest: async () => { + throw denied; + }, + }); + try { + const result = await service.inspectWithObjdump({ + path, + operation: "symbols", + }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "ProviderAdapterError" }, + }); + if (result.ok) return; + expect(result.error.cause).toBe(denied); + } finally { + await service.close(); + await rm(directory, { recursive: true, force: true }); + } +}); + +it.each([ + { + captured: true, + code: "EACCES", + tag: "AnalysisAccessDeniedError", + }, + { + captured: true, + code: "ETIMEDOUT", + tag: "AnalysisTimeoutError", + }, + { + captured: true, + code: "ABORT_ERR", + tag: "AnalysisCancelledError", + }, + { + captured: false, + code: "EACCES", + tag: "AnalysisAccessDeniedError", + }, + { + captured: false, + code: "ETIMEDOUT", + tag: "AnalysisTimeoutError", + }, + { + captured: false, + code: "ABORT_ERR", + tag: "AnalysisCancelledError", + }, +] as const)( + "classifies a $code command failure with captured output $captured while the caller is aborted", + async ({ captured, code, tag }) => { + const directory = await mkdtemp(join(tmpdir(), "rea-reverse-command-abort-")); + const path = join(directory, "fixture.bin"); + await writeFile(path, "fixture"); + const reason = Object.freeze({ requested: `stop-${code}` }); + const controller = new AbortController(); + const failure = Object.assign(new Error("provider failure"), { + code, + ...(captured + ? { stdout: "partial stdout", stderr: "partial stderr" } + : {}), + }); + const service = new TestReverseEngineeringService({ + run: async () => { + controller.abort(reason); + throw failure; + }, + }); + try { + const result = await service.inspectWithObjdump( + { path, operation: "symbols" }, + { signal: controller.signal }, + ); + expect(result).toMatchObject({ ok: false, error: { _tag: tag } }); + if (result.ok) return; + expect(result.error.cause).toBe( + code === "ABORT_ERR" ? reason : failure, + ); + if (captured && code === "ABORT_ERR") { + expect(result.error.capturedOutput).toMatchObject({ + stdout: "partial stdout", + stderr: "partial stderr", + }); + expect(result.error.partialObservation).toMatchObject({ + raw_result: { stdout: "partial stdout" }, + }); + } + } finally { + await service.close(); + await rm(directory, { recursive: true, force: true }); + } + }, +); From de4579e993bde2935a955166f7556d10ebaa3d68 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:17:46 +0800 Subject: [PATCH 10/35] fix(observation): keep incomplete cancellation observations (#5644) * fix(observation): keep incomplete cancellation observations Service projection treated a retained observation that failed the completed schema as a malformed completed result. Cancellation became unreadable_output and the partial was dropped. Incomplete observations now stay on the owner error, including the exact abort reason. * fix(observation): drop non-JSON incomplete observations Keep a retained partial only when projectAnalysisError can serialize it. Clear a non-JSON partial and record an AnalysisOutputError as a secondary failure so cancellation and timeout stay the owner error. Co-authored-by: morluto * fix(observation): validate incomplete partials without copying jsonValueSchema.safeParse cloned a JSON-safe partial that projection copies again. Use jsonValueValidationIssue so the gate only checks. Co-authored-by: morluto * test(ida): expect non-retryable unresolved procedure lookups Main stops treating invalid_request as retryable. An unchanged missing procedure stays invalid_request and is not marked retryable. Co-authored-by: morluto * test(observation): share incomplete-observation assertions The new non-JSON cases repeated the same owner-error checks and pushed duplication over the 3.5% limit. One expectation helper keeps those checks without a second copy. Co-authored-by: morluto * test(observation): keep the assertion helper out of the package The helper imports Vitest. Rename it to a fixture so the production build excludes it and dist does not import a dev dependency. Co-authored-by: morluto * fix(observation): clear non-JSON partials on cleanup primaries Projection can return the cleanup primary after retry. Clearing only the wrapper left that primary's partial in place, so projectAnalysisError threw. The gate now clears every non-JSON partial in the chain. Co-authored-by: morluto * fix(observation): freeze retained incomplete observations A JSON partial stayed a live provider object, so a later mutation could make projection throw. The gate now keeps one frozen snapshot and still drops a non-JSON partial without copying it. Co-authored-by: morluto * refactor(observation): share web runtime partial retention Observe and inspect used the same promote-or-keep path. One helper owns that sequence so the two operations cannot drift. Co-authored-by: morluto * fix(observation): snapshot a shared partial once A cleanup wrapper and its primary can hold the same partial object. The gate now reuses the first frozen snapshot for that object. Co-authored-by: morluto * fix(observation): parse a kept partial once jsonValueSchema.safeParse validates and copies only a JSON partial. The gate freezes that copy and no longer walks the same value first. Co-authored-by: morluto * test(observation): share repeated observation fixtures The incomplete-observation tests repeated the same provider setup and an empty Electron coverage tail, which pushed duplication over 3.5%. One coverage fixture and shared assertions bring the tree back under the threshold. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- .../BrowserObservationService.test.ts | 185 ++++++++++++++++-- src/application/BrowserObservationService.ts | 94 ++++----- .../BrowserScenarioCaptureService.test.ts | 144 ++++++++++---- .../BrowserScenarioCaptureService.ts | 24 +-- .../ProviderOperationExecution.test.ts | 95 ++++++++- src/application/ProviderOperationExecution.ts | 53 +++++ src/application/WebRuntimeService.test.ts | 135 ++++++++++++- src/application/WebRuntimeService.ts | 112 +++++------ .../ElectronObservationService.test.ts | 98 ++++++---- .../javascript/ElectronObservationService.ts | 53 ++--- ...avaScriptRuntimeObservationService.test.ts | 65 ++++-- .../JavaScriptRuntimeObservationService.ts | 25 +-- .../ownerObservationExpectation.fixture.ts | 72 +++++++ .../electronPageInspection.fixture.ts | 34 ++++ tests/boundary/mcp/electronMcp.test.ts | 30 +-- 15 files changed, 915 insertions(+), 304 deletions(-) create mode 100644 src/application/ownerObservationExpectation.fixture.ts create mode 100644 src/domain/javascript/electronPageInspection.fixture.ts diff --git a/src/application/BrowserObservationService.test.ts b/src/application/BrowserObservationService.test.ts index d2cdde1c27..b6e33a0e23 100644 --- a/src/application/BrowserObservationService.test.ts +++ b/src/application/BrowserObservationService.test.ts @@ -3,9 +3,19 @@ import type { BrowserObservationPort } from "./BrowserObservationPort.js"; import { listBrowserTargets } from "./BrowserObservationService.js"; import type { BrowserTargetList } from "../domain/browserObservationSchemas.js"; import { parseEvidence } from "../domain/evidence.js"; -import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { + expectDroppedPartial, + expectKeptPartial, + expectOutputError, + expectOwnerError, +} from "./ownerObservationExpectation.fixture.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; -import { ok } from "../domain/result.js"; +import { err, ok } from "../domain/result.js"; import { passiveNetworkCapture } from "../../tests/fixtures/passiveNetworkCapture.js"; import { inspectWebPage } from "./BrowserObservationService.js"; import { analyzeWebBundleInputSchema } from "../domain/webBundleAnalysis.js"; @@ -71,11 +81,7 @@ it("does not let late cancellation hide invalid completed browser output", async provider(async () => ok(malformed)), input, ); - expect(output).toMatchObject({ - ok: false, - error: { _tag: "AnalysisOutputError" }, - }); - if (!output.ok) expect(output.error).toBeInstanceOf(AnalysisOutputError); + expectOutputError(output); const controller = new AbortController(); const late = await listBrowserTargets( @@ -86,11 +92,7 @@ it("does not let late cancellation hide invalid completed browser output", async input, { signal: controller.signal }, ); - expect(late).toMatchObject({ - ok: false, - error: { _tag: "AnalysisOutputError" }, - }); - if (!late.ok) expect(late.error).toBeInstanceOf(AnalysisOutputError); + expectOutputError(late); }); it("converts a raw page inspection retained after browser cleanup failure", async () => { @@ -215,3 +217,162 @@ it("preserves a prior caller reason without admitting browser work", async () => expect(result.error._tag).toBe("AnalysisCancelledError"); expect(result.error.cause).toBe(reason); }); + +it("keeps page cancellation when the retained inspection is incomplete", async () => { + const reason = { requested: "stop during page inspection" }; + const partial = { + target: { target_id: "page" }, + network: {}, + capture_window: {}, + }; + const failure = new AnalysisCancelledError("inspect_web_page", { + cause: reason, + partialObservation: partial as never, + }); + const browser = { + ...provider(async () => ok(report)), + inspectPage: async () => err(failure), + } as unknown as BrowserObservationPort; + const pageInput = inspectWebPageInputSchema.parse({ + cdp_endpoint: input.cdp_endpoint.recorded, + allowed_origins: [], + target_id: "page", + observation_ms: 0, + include_script_sources: false, + }); + const result = await inspectWebPage(browser, pageInput); + expectKeptPartial(expectOwnerError(result, failure, reason), partial); +}); + +it("keeps bundle cancellation and an earlier failure when retained facts are incomplete", async () => { + const reason = { requested: "stop during bundle analysis" }; + const partial = { observations: [], capture: { status: "partial" } }; + const cancelled = new AnalysisCancelledError("analyze_web_bundle", { + cause: reason, + partialObservation: partial as never, + }); + const timedOut = new BrowserObservationError( + "analyze_web_bundle", + "timeout", + { partialObservation: partial as never }, + ); + const selected = analyzeWebBundleInputSchema.parse({ + cdp_endpoint: input.cdp_endpoint.recorded, + target_id: "page", + observation_ms: 0, + }); + for (const failure of [cancelled, timedOut]) { + const browser = { + ...provider(async () => ok(report)), + analyzeBundle: async () => err(failure), + } as unknown as BrowserObservationPort; + const result = await analyzeWebBundle(browser, selected); + expectKeptPartial( + expectOwnerError(result, failure, failure.cause), + partial, + failure === timedOut ? "provider_timeout" : "cancelled", + ); + } + expect(projectAnalysisError(cancelled).code).toBe("cancelled"); +}); + +const pageInput = () => + inspectWebPageInputSchema.parse({ + cdp_endpoint: input.cdp_endpoint.recorded, + allowed_origins: [], + target_id: "page", + observation_ms: 0, + include_script_sources: false, + }); + +const bundleInput = () => + analyzeWebBundleInputSchema.parse({ + cdp_endpoint: input.cdp_endpoint.recorded, + target_id: "page", + observation_ms: 0, + }); + +const browserReturning = ( + failure: BrowserObservationError | AnalysisCancelledError, +) => + ({ + ...provider(async () => ok(report)), + inspectPage: async () => err(failure), + analyzeBundle: async () => err(failure), + }) as unknown as BrowserObservationPort; + +it.each([ + { + name: "bundle bytes", + operation: "analyze_web_bundle" as const, + partial: { observations: [], capture: { bytes: Number.NaN } }, + }, + { + name: "bundle observations", + operation: "analyze_web_bundle" as const, + partial: { observations: [undefined], capture: {} }, + }, + { + name: "page requests", + operation: "inspect_web_page" as const, + partial: { + target: { target_id: "page" }, + network: { requests: Number.NaN }, + capture_window: {}, + }, + }, +])( + "drops a non-JSON $name partial and keeps cancellation primary", + async ({ operation, partial }) => { + const reason = { requested: `stop ${operation}` }; + const failure = new AnalysisCancelledError(operation, { + cause: reason, + partialObservation: partial as never, + }); + const result = + operation === "inspect_web_page" + ? await inspectWebPage(browserReturning(failure), pageInput()) + : await analyzeWebBundle(browserReturning(failure), bundleInput()); + const error = expectOwnerError(result, failure, reason); + expectDroppedPartial( + error, + "Browser provider retained a non-JSON incomplete observation.", + ); + expect(projectAnalysisError(error).details).not.toHaveProperty( + "partial_observation", + ); + }, +); + +it.each([ + { + name: "schema-shaped page", + partial: { + target: { target_id: "page" }, + network: { requests: Number.NaN }, + capture_window: {}, + }, + }, + { + name: "shapeless page", + partial: { requests: Number.NaN }, + }, +])( + "drops a non-JSON $name partial and keeps timeout primary", + async ({ partial }) => { + const failure = new BrowserObservationError("inspect_web_page", "timeout", { + partialObservation: partial as never, + }); + const error = expectOwnerError( + await inspectWebPage(browserReturning(failure), pageInput()), + failure, + undefined, + ); + expectDroppedPartial( + error, + "Browser provider retained a non-JSON incomplete observation.", + "provider_timeout", + ); + expect(projectAnalysisError(error).category).toBe("timeout"); + }, +); diff --git a/src/application/BrowserObservationService.ts b/src/application/BrowserObservationService.ts index 6251e9d4dd..1e32b4ead4 100644 --- a/src/application/BrowserObservationService.ts +++ b/src/application/BrowserObservationService.ts @@ -25,9 +25,11 @@ import type { ExecutionOptions } from "./AnalysisProvider.js"; import type { BrowserObservationPort } from "./BrowserObservationPort.js"; import { createBrowserEvidence } from "./BrowserEvidence.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; -import { completeProviderEvidence } from "./ProviderOperationExecution.js"; +import { + completeProviderEvidence, + keepIncompleteObservation, +} from "./ProviderOperationExecution.js"; import { webPageInspectionSchema } from "../domain/browserObservationSchemas.js"; -import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; /** List browser targets within the endpoint and origin scope in the request. */ export const listBrowserTargets = async ( @@ -275,7 +277,10 @@ const browserEvidence = ( ); }; -/** Retain the completed capture or bundle facts under their actual operation. */ +const NON_JSON_BROWSER_OBSERVATION = + "Browser provider retained a non-JSON incomplete observation."; + +/** Promote completed page or bundle facts; keep only a JSON-safe incomplete observation. */ const retainBrowserObservation = ( error: AnalysisError, input: InspectWebPageInput | AnalyzeWebBundleInput, @@ -291,59 +296,42 @@ const retainBrowserObservation = ( "capture" in partial ) { const bundle = webBundleAnalysisSchema.safeParse(partial); - if (!bundle.success) { - const malformed = new AnalysisOutputError( - errorOperation, - "Browser provider retained malformed completed bundle facts.", - { cause: bundle.error }, - ); - return err( - error.cleanupIncomplete - ? analysisErrorWithCleanupFailure( - malformed, - error, - errorOperation, - ).bindCleanupDependencies([malformed, error]) - : malformed, + if (bundle.success) { + error.partialObservation = createBrowserEvidence( + "analyze_web_bundle", + input, + bundle.data, + provider.identity(), ); + return err(error); } - error.partialObservation = createBrowserEvidence( - "analyze_web_bundle", - input, - bundle.data, - provider.identity(), - ); - return err(error); - } - if ( - partial === undefined || - "evidence_id" in partial || - !("target" in partial) || - !("network" in partial) || - !("capture_window" in partial) - ) - return err(error); - const inspection = webPageInspectionSchema.safeParse(partial); - if (!inspection.success) { - if (error instanceof AnalysisCancelledError) - return err( - new AnalysisOutputError( - errorOperation, - "Browser provider retained a malformed completed page inspection.", - { cause: inspection.error }, - ), + } else if ( + partial !== undefined && + !("evidence_id" in partial) && + "target" in partial && + "network" in partial && + "capture_window" in partial + ) { + const inspection = webPageInspectionSchema.safeParse(partial); + if (inspection.success) { + const pageInput: InspectWebPageInput = + "fetch_source_maps" in input + ? (({ fetch_source_maps: _fetchSourceMaps, ...page }) => page)(input) + : input; + error.partialObservation = createBrowserEvidence( + "inspect_web_page", + pageInput, + inspection.data, + provider.identity(), ); - return err(error); + return err(error); + } } - const pageInput: InspectWebPageInput = - "fetch_source_maps" in input - ? (({ fetch_source_maps: _fetchSourceMaps, ...page }) => page)(input) - : input; - error.partialObservation = createBrowserEvidence( - "inspect_web_page", - pageInput, - inspection.data, - provider.identity(), + return err( + keepIncompleteObservation( + error, + errorOperation, + NON_JSON_BROWSER_OBSERVATION, + ), ); - return err(error); }; diff --git a/src/application/BrowserScenarioCaptureService.test.ts b/src/application/BrowserScenarioCaptureService.test.ts index df187682a4..34a3c588cb 100644 --- a/src/application/BrowserScenarioCaptureService.test.ts +++ b/src/application/BrowserScenarioCaptureService.test.ts @@ -1,13 +1,26 @@ import { expect, it } from "vitest"; -import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; import { parseEvidence } from "../domain/evidence.js"; import { browserScenarioInputSchema } from "../domain/browserScenario.js"; -import { err } from "../domain/result.js"; +import type { + BrowserScenarioCapture, + BrowserScenarioPartialObservation, +} from "../domain/browserScenarioCapture.js"; +import { err, ok } from "../domain/result.js"; import { scriptScenarioFixture } from "../../tests/fixtures/webScriptCapture.js"; import type { BrowserScenarioCapturePort } from "./BrowserScenarioCapturePort.js"; import { captureBrowserScenario } from "./BrowserScenarioCaptureService.js"; +import { + expectDroppedPartial, + expectKeptPartial, + expectOwnerError, +} from "./ownerObservationExpectation.fixture.js"; const scenario = browserScenarioInputSchema.parse({ browser: { mode: "launch", executable_path: "/fixture/chrome" }, @@ -15,6 +28,34 @@ const scenario = browserScenarioInputSchema.parse({ actions: [{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 }], }); const capture = scriptScenarioFixture(); +const scenarioIdentity = { + id: "scenario-fixture", + name: "Scenario fixture", + version: "1", +}; +const runScenario = ( + failure: AnalysisError, + id = { id: "fixture", name: "fixture", version: "1" }, +) => + captureBrowserScenario( + { + identity: () => id, + captureScenario: async () => err(failure), + }, + scenario, + ); +const expectRetainedCapture = ( + result: Awaited>, + failure: AnalysisError, +) => { + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected the retained capture"); + expect(result.error).toBe(failure); + expect(parseEvidence(result.error.partialObservation)).toMatchObject({ + operation: "capture_browser_scenario", + normalized_result: capture, + }); +}; it("converts a retained raw scenario capture into partial Evidence", async () => { const failure = new AnalysisCancelledError("capture_browser_scenario", { @@ -23,23 +64,7 @@ it("converts a retained raw scenario capture into partial Evidence", async () => capture, }, }); - const provider: BrowserScenarioCapturePort = { - identity: () => ({ - id: "scenario-fixture", - name: "Scenario fixture", - version: "1", - }), - captureScenario: async () => err(failure), - }; - const result = await captureBrowserScenario(provider, scenario); - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error).toBe(failure); - expect(parseEvidence(result.error.partialObservation)).toMatchObject({ - operation: "capture_browser_scenario", - normalized_result: capture, - }); - } + expectRetainedCapture(await runScenario(failure, scenarioIdentity), failure); }); it("converts a raw scenario capture retained after cleanup failure", async () => { @@ -57,24 +82,9 @@ it("converts a raw scenario capture retained after cleanup failure", async () => }, }, ); - const provider: BrowserScenarioCapturePort = { - identity: () => ({ - id: "scenario-fixture", - name: "Scenario fixture", - version: "1", - }), - captureScenario: async () => err(failure), - }; - const result = await captureBrowserScenario(provider, scenario); - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error).toBe(failure); - expect(result.error.cleanupIncomplete).toBe(true); - expect(parseEvidence(result.error.partialObservation)).toMatchObject({ - operation: "capture_browser_scenario", - normalized_result: capture, - }); - } + const result = await runScenario(failure, scenarioIdentity); + expectRetainedCapture(result, failure); + if (!result.ok) expect(result.error.cleanupIncomplete).toBe(true); }); it.each([ @@ -103,3 +113,63 @@ it.each([ expect(calls).toBe(0); }, ); + +it("keeps startup cancellation and its incomplete capture", async () => { + const reason = { requested: "stop during initial navigation" }; + const completed = scriptScenarioFixture(); + const { + completeness: _completeness, + steps: _steps, + browser, + ...retained + } = completed; + const partial: BrowserScenarioPartialObservation = { + kind: "browser-scenario-observation", + capture: { + ...retained, + browser: { ...browser, cleanup: "incomplete" }, + steps: [], + }, + }; + const failure = new AnalysisCancelledError("capture_browser_scenario", { + cause: reason, + partialObservation: partial, + }); + const result = await runScenario(failure); + expectKeptPartial(expectOwnerError(result, failure, reason), partial); +}); + +it.each([{ steps: [undefined] }, { started_at: Number.NaN }])( + "drops a non-JSON incomplete scenario capture and keeps cancellation primary", + async (capturePartial) => { + const reason = { requested: "stop during initial navigation" }; + const partial: BrowserScenarioPartialObservation = { + kind: "browser-scenario-observation", + capture: capturePartial as never, + }; + const failure = new AnalysisCancelledError("capture_browser_scenario", { + cause: reason, + partialObservation: partial, + }); + const result = await runScenario(failure); + expectDroppedPartial( + expectOwnerError(result, failure, reason), + "Browser scenario provider retained a non-JSON incomplete observation.", + ); + }, +); + +it("rejects a successful capture that is not a completed scenario", async () => { + const provider: BrowserScenarioCapturePort = { + identity: () => ({ id: "fixture", name: "fixture", version: "1" }), + captureScenario: async () => ok({} as BrowserScenarioCapture), + }; + const result = await captureBrowserScenario(provider, scenario); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected a completed-capture refusal"); + expect(result.error).toBeInstanceOf(AnalysisOutputError); + expect(result.error.message).toContain( + "returned a malformed completed capture", + ); + expect(result.error.partialObservation).toBeUndefined(); +}); diff --git a/src/application/BrowserScenarioCaptureService.ts b/src/application/BrowserScenarioCaptureService.ts index 0a5bc26019..24e8c0ca4f 100644 --- a/src/application/BrowserScenarioCaptureService.ts +++ b/src/application/BrowserScenarioCaptureService.ts @@ -11,7 +11,10 @@ import type { Evidence } from "../domain/evidence.js"; import type { BrowserScenario } from "../domain/browserScenario.js"; import { err, type Result } from "../domain/result.js"; import { browserScenarioCaptureSchema } from "../domain/browserScenarioCapture.js"; -import { completeProviderEvidence } from "./ProviderOperationExecution.js"; +import { + completeProviderEvidence, + keepIncompleteObservation, +} from "./ProviderOperationExecution.js"; const OPERATION = "capture_browser_scenario" as const; const PROVIDER_ID = "rea-playwright-browser-scenario"; @@ -43,6 +46,8 @@ export const captureBrowserScenario = async ( partial.kind === "browser-scenario-observation" ) { const report = browserScenarioCaptureSchema.safeParse(partial.capture); + // Only a completed capture becomes Evidence. A JSON-safe incomplete + // observation stays on the owner error, including typed cancellation. if ( report.success && report.data.steps.every(({ status }) => status === "completed") @@ -52,19 +57,16 @@ export const captureBrowserScenario = async ( report.data, provider.identity(), ); - } else if (!report.success) { - if (captured.error instanceof AnalysisCancelledError) - return err( - new AnalysisOutputError( - OPERATION, - "Browser scenario provider retained a malformed completed capture.", - { cause: report.error }, - ), - ); return captured; } } - return captured; + return err( + keepIncompleteObservation( + captured.error, + OPERATION, + "Browser scenario provider retained a non-JSON incomplete observation.", + ), + ); } const report = browserScenarioCaptureSchema.safeParse(captured.value); if (!report.success) diff --git a/src/application/ProviderOperationExecution.test.ts b/src/application/ProviderOperationExecution.test.ts index 7729cb42b7..f3a36e1b5a 100644 --- a/src/application/ProviderOperationExecution.test.ts +++ b/src/application/ProviderOperationExecution.test.ts @@ -1,8 +1,18 @@ import { expect, it, vi } from "vitest"; import { z } from "zod"; -import { AnalysisInputError } from "../domain/analysisErrorCore.js"; -import { executeProviderOperation } from "./ProviderOperationExecution.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { + executeProviderOperation, + keepIncompleteObservation, +} from "./ProviderOperationExecution.js"; it("projects request-schema issues without exposing the request envelope", async () => { const execute = vi.fn(); @@ -87,3 +97,84 @@ it("preserves operation correction outside the input envelope", async () => { ]); expect(execute).not.toHaveBeenCalled(); }); + +const cancelledScenario = (partial: unknown) => + new AnalysisCancelledError("capture_browser_scenario", { + cause: { requested: "stop during capture" }, + partialObservation: partial as never, + }); + +const wrapScenarioCleanup = (primary: AnalysisCancelledError) => + analysisErrorWithCleanupFailure( + primary, + new ProviderCleanupError("fixture", ["browser"], {}), + "capture_browser_scenario", + ); + +it("clears a non-JSON partial from the cleanup primary projection returns after retry", () => { + const partial = { steps: [] as unknown[] }; + const primary = cancelledScenario(partial); + const composed = wrapScenarioCleanup(primary); + partial.steps.push(undefined); + const reason = + "Browser scenario provider retained a non-JSON incomplete observation."; + const retained = keepIncompleteObservation( + composed, + "capture_browser_scenario", + reason, + ); + + expect(retained).toBe(composed); + expect(composed.partialObservation).toBeUndefined(); + expect(primary.partialObservation).toBeUndefined(); + expect(composed.secondaryFailures).toMatchObject([ + { _tag: "AnalysisOutputError", reason }, + ]); + expect(primary.secondaryFailures[0]).toBeInstanceOf(AnalysisOutputError); + expect(projectAnalysisError(composed).code).toBe("cleanup_incomplete"); + + composed.resolveCleanup(); + expect(projectAnalysisError(composed)).toMatchObject({ + code: "cancelled", + category: "cancelled", + }); + expect(projectAnalysisError(composed).details).not.toHaveProperty( + "partial_observation", + ); +}); + +it("freezes one shared snapshot for a cleanup wrapper and its primary", () => { + const partial = { steps: ["kept"] }; + const primary = cancelledScenario(partial); + const composed = wrapScenarioCleanup(primary); + keepIncompleteObservation( + composed, + "capture_browser_scenario", + "unused because the partial is JSON", + ); + + expect(composed.partialObservation).toBe(primary.partialObservation); + expect(composed.partialObservation).toEqual({ steps: ["kept"] }); + expect(Object.isFrozen(composed.partialObservation)).toBe(true); + expect(composed.secondaryFailures).toHaveLength(0); + expect(primary.secondaryFailures).toHaveLength(0); +}); + +it("freezes a retained partial so a later mutation cannot break projection", () => { + const partial = { steps: [] as unknown[] }; + const primary = cancelledScenario(partial); + keepIncompleteObservation( + primary, + "capture_browser_scenario", + "unused because the partial is JSON", + ); + partial.steps.push(undefined); + + expect(primary.partialObservation).toEqual({ steps: [] }); + expect(Object.isFrozen(primary.partialObservation)).toBe(true); + expect(primary.secondaryFailures).toHaveLength(0); + expect(projectAnalysisError(primary)).toMatchObject({ + code: "cancelled", + details: { partial_observation: { steps: [] } }, + }); +}); diff --git a/src/application/ProviderOperationExecution.ts b/src/application/ProviderOperationExecution.ts index 7a48e9ae88..38012d0b5b 100644 --- a/src/application/ProviderOperationExecution.ts +++ b/src/application/ProviderOperationExecution.ts @@ -2,12 +2,19 @@ import { z } from "zod"; import type { AnalysisError, AnalysisErrorOptions, + AnalysisPartialObservation, } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, AnalysisInputError, + AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import type { Evidence } from "../domain/evidence.js"; +import { + freezeJsonSnapshot, + isImmutableJsonSnapshot, +} from "../domain/immutableJson.js"; +import { jsonValueSchema } from "../domain/jsonValue.js"; import { projectInputIssues } from "../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -92,3 +99,49 @@ export const completeProviderEvidence = ( ); return ok(evidence); }; + +/** + * Retain an incomplete observation when projection can serialize it. + * A JSON partial is replaced by one frozen snapshot, so a later mutation + * cannot make projection throw and projection does not copy it again. + * A partial that is not JSON is cleared from the returned error and from + * every cleanup primary projection can select later. Each cleared error + * records an output failure, leaving cancellation or timeout primary. + */ +export const keepIncompleteObservation = ( + error: AnalysisError, + operation: string, + reason: string, +): AnalysisError => { + const seen = new Set(); + // One source object can sit on a wrapper and its cleanup primary. + const judged = new Map(); + let current: AnalysisError | undefined = error; + while (current !== undefined && !seen.has(current)) { + seen.add(current); + const partial = current.partialObservation; + if ( + partial !== undefined && + !isImmutableJsonSnapshot(partial) && + typeof partial === "object" + ) { + // safeParse clones only after validation succeeds. Freeze that copy once. + let snapshot = judged.get(partial); + if (!judged.has(partial)) { + const parsed = jsonValueSchema.safeParse(partial); + snapshot = parsed.success + ? (freezeJsonSnapshot(parsed.data) as AnalysisPartialObservation) + : undefined; + judged.set(partial, snapshot); + } + if (snapshot === undefined) { + current.partialObservation = undefined; + current.retainSecondaryFailure( + new AnalysisOutputError(operation, reason), + ); + } else current.partialObservation = snapshot; + } + current = current.cleanupPrimary; + } + return error; +}; diff --git a/src/application/WebRuntimeService.test.ts b/src/application/WebRuntimeService.test.ts index 7e7c651486..f28be157d9 100644 --- a/src/application/WebRuntimeService.test.ts +++ b/src/application/WebRuntimeService.test.ts @@ -6,10 +6,20 @@ import { webListenersFixture, } from "./WebRuntimeService.fixture.js"; import { err, ok } from "../domain/result.js"; -import { AnalysisTimeoutError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, + AnalysisTimeoutError, +} from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { + expectDroppedPartial, + expectKeptPartial, + expectOwnerError, +} from "./ownerObservationExpectation.fixture.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { parseEvidence } from "../domain/evidence.js"; -import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; it("rejects malformed input and prior cancellation without calling a provider", async () => { const provider = recordingWebRuntimePort(); @@ -83,11 +93,7 @@ it("converts raw execution retained by provider cleanup failure into partial Evi { operation: "observe_web_execution", partialObservation: execution }, ); const result = await new WebRuntimeService({ - identity: () => ({ - id: "web-runtime", - name: "Runtime fixture", - version: "1", - }), + identity: runtimeIdentity, observeExecution: async () => err(failure), inspectEventListeners: async () => err(failure), }).observe(args); @@ -161,3 +167,118 @@ it("retains completed execution Evidence on late abort and keeps invalid output }); if (!invalid.ok) expect(invalid.error).toBeInstanceOf(AnalysisOutputError); }); + +const runtimeIdentity = () => ({ + id: "web-runtime", + name: "Runtime fixture", + version: "1", +}); + +const callRuntime = ( + operation: "observe" | "inspect", + failure: AnalysisError, +) => { + const service = new WebRuntimeService({ + identity: runtimeIdentity, + observeExecution: async () => err(failure), + inspectEventListeners: async () => err(failure), + }); + return operation === "observe" + ? service.observe({ + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "selected-page", + observation_ms: 5, + }) + : service.inspect({ + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "selected-page", + selector: "#selected", + }); +}; + +it.each([ + { + operation: "observe" as const, + partial: { + instrumentation: { cleanup: "unconfirmed" }, + window: { requested_ms: 5 }, + }, + }, + { + operation: "inspect" as const, + partial: { + selected_node: { selector: "#selected" }, + listeners: [], + }, + }, +])( + "keeps $operation cancellation when the retained observation is incomplete", + async ({ operation, partial }) => { + const reason = { requested: `stop incomplete ${operation}` }; + const failure = new AnalysisCancelledError( + operation === "observe" + ? "observe_web_execution" + : "inspect_web_event_listeners", + { cause: reason, partialObservation: partial as never }, + ); + expectKeptPartial( + expectOwnerError(await callRuntime(operation, failure), failure, reason), + partial, + ); + }, +); + +it.each([ + { + operation: "observe" as const, + partial: { + instrumentation: { cleanup: "unconfirmed" }, + window: { requested_ms: Number.NaN }, + }, + reason: "Provider retained a non-JSON incomplete execution observation.", + }, + { + operation: "inspect" as const, + partial: { + selected_node: { selector: "#selected" }, + listeners: [undefined], + }, + reason: + "Provider retained a non-JSON incomplete event-listener observation.", + }, +])( + "drops a non-JSON $operation partial and keeps cancellation primary", + async ({ operation, partial, reason: outputReason }) => { + const reason = { requested: `stop unsafe ${operation}` }; + const failure = new AnalysisCancelledError( + operation === "observe" + ? "observe_web_execution" + : "inspect_web_event_listeners", + { cause: reason, partialObservation: partial as never }, + ); + expectDroppedPartial( + expectOwnerError(await callRuntime(operation, failure), failure, reason), + outputReason, + ); + }, +); + +it("drops a non-JSON execution partial and keeps timeout primary", async () => { + const failure = new AnalysisTimeoutError("observe_web_execution", 5, { + partialObservation: { + instrumentation: { samples: [undefined] }, + window: { requested_ms: 5 }, + } as never, + }); + const error = expectOwnerError( + await callRuntime("observe", failure), + failure, + undefined, + ); + expectDroppedPartial( + error, + "Provider retained a non-JSON incomplete execution observation.", + "provider_timeout", + ); + expect(projectAnalysisError(error).category).toBe("timeout"); +}); diff --git a/src/application/WebRuntimeService.ts b/src/application/WebRuntimeService.ts index 38d2934545..627ada4906 100644 --- a/src/application/WebRuntimeService.ts +++ b/src/application/WebRuntimeService.ts @@ -18,7 +18,10 @@ import { inspectWebEventListenersInputSchema, webEventListenersSchema, } from "../domain/webEventListeners.js"; -import { completeProviderEvidence } from "./ProviderOperationExecution.js"; +import { + completeProviderEvidence, + keepIncompleteObservation, +} from "./ProviderOperationExecution.js"; /** Shared public workflows keep instrumentation and listener inspection independently selectable. */ export class WebRuntimeService { @@ -38,35 +41,19 @@ export class WebRuntimeService { ); if (!parsed.ok) return parsed; const result = await this.provider.observeExecution(parsed.value, options); - if (!result.ok) { - const partial = result.error.partialObservation; - if ( - partial !== undefined && - !("evidence_id" in partial) && - "instrumentation" in partial && - "window" in partial - ) { - const verifiedPartial = webExecutionSchema.safeParse(partial); - if (verifiedPartial.success) { - const evidence = this.evidence( - operation, - parsed.value, - verifiedPartial.data, - ); - if (evidence.ok) result.error.partialObservation = evidence.value; - else if (result.error instanceof AnalysisCancelledError) - return evidence; - } else if (result.error instanceof AnalysisCancelledError) - return err( - new AnalysisOutputError( - operation, - "Provider retained a malformed completed execution observation.", - { cause: verifiedPartial.error }, - ), - ); - } - return result; - } + if (!result.ok) + return this.retainRuntimePartial( + result.error, + operation, + "Provider retained a non-JSON incomplete execution observation.", + (partial) => { + if (!("instrumentation" in partial) || !("window" in partial)) + return undefined; + const verified = webExecutionSchema.safeParse(partial); + if (!verified.success) return undefined; + return this.evidence(operation, parsed.value, verified.data); + }, + ); const verified = webExecutionSchema.safeParse(result.value); if (!verified.success) return err(new AnalysisOutputError(operation, verified.error.message)); @@ -100,35 +87,19 @@ export class WebRuntimeService { parsed.value, options, ); - if (!result.ok) { - const partial = result.error.partialObservation; - if ( - partial !== undefined && - !("evidence_id" in partial) && - "selected_node" in partial && - "listeners" in partial - ) { - const verifiedPartial = webEventListenersSchema.safeParse(partial); - if (verifiedPartial.success) { - const evidence = this.evidence( - operation, - parsed.value, - verifiedPartial.data, - ); - if (evidence.ok) result.error.partialObservation = evidence.value; - else if (result.error instanceof AnalysisCancelledError) - return evidence; - } else if (result.error instanceof AnalysisCancelledError) - return err( - new AnalysisOutputError( - operation, - "Provider retained malformed completed event-listener data.", - { cause: verifiedPartial.error }, - ), - ); - } - return result; - } + if (!result.ok) + return this.retainRuntimePartial( + result.error, + operation, + "Provider retained a non-JSON incomplete event-listener observation.", + (partial) => { + if (!("selected_node" in partial) || !("listeners" in partial)) + return undefined; + const verified = webEventListenersSchema.safeParse(partial); + if (!verified.success) return undefined; + return this.evidence(operation, parsed.value, verified.data); + }, + ); const verified = webEventListenersSchema.safeParse(result.value); if (!verified.success) return err(new AnalysisOutputError(operation, verified.error.message)); @@ -145,6 +116,29 @@ export class WebRuntimeService { return this.evidence(operation, parsed.value, verified.data, options); } + /** Promote a schema-valid partial, or keep a JSON snapshot of an incomplete one. */ + private retainRuntimePartial( + error: AnalysisError, + operation: string, + nonJsonReason: string, + promote: ( + partial: NonNullable, + ) => Result | undefined, + ): Result { + const partial = error.partialObservation; + if (partial !== undefined && !("evidence_id" in partial)) { + const evidence = promote(partial); + if (evidence !== undefined) { + if (evidence.ok) { + error.partialObservation = evidence.value; + return err(error); + } + if (error instanceof AnalysisCancelledError) return evidence; + } + } + return err(keepIncompleteObservation(error, operation, nonJsonReason)); + } + private evidence( operation: string, input: diff --git a/src/application/javascript/ElectronObservationService.test.ts b/src/application/javascript/ElectronObservationService.test.ts index 0d055e1f09..01e9e8c0e2 100644 --- a/src/application/javascript/ElectronObservationService.test.ts +++ b/src/application/javascript/ElectronObservationService.test.ts @@ -1,6 +1,5 @@ import { expect, it } from "vitest"; -import { classifyBrowserCompleteness } from "../../domain/browserCompleteness.js"; import { AnalysisCancelledError, AnalysisOutputError, @@ -12,14 +11,27 @@ import { electronTargetListSchema, inspectElectronPageInputSchema, } from "../../domain/javascript/electronObservation.js"; +import { emptyElectronPageCoverage } from "../../domain/javascript/electronPageInspection.fixture.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { err, ok } from "../../domain/result.js"; import type { ElectronObservationPort } from "./ElectronObservationPort.js"; import { inspectElectronPage, listElectronTargets, } from "./ElectronObservationService.js"; +import { + expectDroppedPartial, + expectKeptPartial, + expectOwnerError, +} from "../ownerObservationExpectation.fixture.js"; const identity = { id: "electron-fixture", name: "Fixture", version: "1" }; +const pageProvider = (failure: AnalysisError): ElectronObservationPort => ({ + identity: () => identity, + listTargets: async () => + err(new AnalysisCancelledError("list_electron_targets")), + inspectPage: async () => err(failure), +}); const input = inspectElectronPageInputSchema.parse({ cdp_endpoint: "http://127.0.0.1:9222", target_id: "electron-page", @@ -40,33 +52,7 @@ const inspection = electronPageInspectionSchema.parse({ file_path: "/app/index.html", attached: false, }, - capture_window: { - started_at: "2026-01-01T00:00:00.000Z", - ended_at: "2026-01-01T00:00:00.000Z", - observation_ms: 0, - }, - completeness: classifyBrowserCompleteness({ - policyFilteredSections: new Set(), - attachLimitedSections: new Set(), - truncatedSections: new Set(), - unavailableSections: new Set(), - excluded: [], - droppedEvents: { - scripts: 0, - network_requests: 0, - console_events: 0, - websocket_connections: 0, - websocket_frames: 0, - webmcp_tools: 0, - timeline_events: 0, - }, - }), - frames: [], - dom: { total_nodes: 0, nodes: [] }, - scripts: { total: 0, items: [] }, - resources: [], - workers: [], - limitations: [], + ...emptyElectronPageCoverage(), }); it("converts a retained raw Electron inspection into partial Evidence", async () => { @@ -77,13 +63,7 @@ it("converts a retained raw Electron inspection into partial Evidence", async () cleanup: "transport-closed", }, }); - const provider: ElectronObservationPort = { - identity: () => identity, - listTargets: async () => - err(new AnalysisCancelledError("list_electron_targets")), - inspectPage: async () => err(failure), - }; - const result = await inspectElectronPage(provider, input); + const result = await inspectElectronPage(pageProvider(failure), input); expect(result.ok).toBe(false); if (!result.ok) { expect(result.error).toBe(failure); @@ -116,13 +96,7 @@ it("converts a raw inspection retained by provider cleanup failure", async () => }, }, ); - const provider: ElectronObservationPort = { - identity: () => identity, - listTargets: async () => - err(new AnalysisCancelledError("list_electron_targets")), - inspectPage: async () => err(failure), - }; - const result = await inspectElectronPage(provider, input); + const result = await inspectElectronPage(pageProvider(failure), input); expect(result.ok).toBe(false); if (!result.ok) { expect(result.error).toBe(failure); @@ -244,3 +218,43 @@ it.each(["list", "inspect"] as const)( expect(providerCalls).toBe(0); }, ); + +it("keeps cancellation when the retained Electron inspection is incomplete", async () => { + const reason = { requested: "stop before page inspection completed" }; + const partial = { + kind: "electron-page-observation", + inspection: { target_id: "electron-page" }, + cleanup: "incomplete", + }; + const failure = new AnalysisCancelledError("inspect_electron_page", { + cause: reason, + partialObservation: partial as never, + }); + const result = await inspectElectronPage(pageProvider(failure), input); + expectKeptPartial(expectOwnerError(result, failure, reason), partial); +}); + +it.each([ + { inspection: { requests: Number.NaN }, cleanup: "incomplete" as const }, + { + inspection: { scripts: [undefined] }, + cleanup: "transport-closed" as const, + }, +])( + "drops a non-JSON Electron inspection and keeps cancellation primary", + async (unsafe) => { + const reason = { requested: "stop before page inspection completed" }; + const failure = new AnalysisCancelledError("inspect_electron_page", { + cause: reason, + partialObservation: { + kind: "electron-page-observation", + ...unsafe, + } as never, + }); + const result = await inspectElectronPage(pageProvider(failure), input); + expectDroppedPartial( + expectOwnerError(result, failure, reason), + "Electron provider retained a non-JSON incomplete observation.", + ); + }, +); diff --git a/src/application/javascript/ElectronObservationService.ts b/src/application/javascript/ElectronObservationService.ts index 67a0776cb4..ead41e418f 100644 --- a/src/application/javascript/ElectronObservationService.ts +++ b/src/application/javascript/ElectronObservationService.ts @@ -18,7 +18,10 @@ import { electronPageInspectionSchema, electronTargetListSchema, } from "../../domain/javascript/electronObservation.js"; -import { completeProviderEvidence } from "../ProviderOperationExecution.js"; +import { + completeProviderEvidence, + keepIncompleteObservation, +} from "../ProviderOperationExecution.js"; /** List local file pages exposed by the explicitly selected endpoint. */ export const listElectronTargets = async ( @@ -79,34 +82,32 @@ export const inspectElectronPage = async ( partial.kind === "electron-page-observation" ) { const report = electronPageInspectionSchema.safeParse(partial.inspection); - if (!report.success) { - if (result.error instanceof AnalysisCancelledError) - return err( - new AnalysisOutputError( - "inspect_electron_page", - "Electron provider retained a malformed completed page inspection.", - { cause: report.error }, - ), - ); + if (report.success) { + const inspection = { + ...report.data, + limitations: [ + ...report.data.limitations, + partial.cleanup === "transport-closed" + ? "REA closed its CDP transport; this does not imply target shutdown." + : "CDP transport cleanup remains incomplete.", + ], + } satisfies ElectronPageInspection; + result.error.partialObservation = createElectronEvidence( + "inspect_electron_page", + input, + inspection, + ready.value.identity(), + ); return result; } - const inspection = { - ...report.data, - limitations: [ - ...report.data.limitations, - partial.cleanup === "transport-closed" - ? "REA closed its CDP transport; this does not imply target shutdown." - : "CDP transport cleanup remains incomplete.", - ], - } satisfies ElectronPageInspection; - result.error.partialObservation = createElectronEvidence( - "inspect_electron_page", - input, - inspection, - ready.value.identity(), - ); } - return result; + return err( + keepIncompleteObservation( + result.error, + "inspect_electron_page", + "Electron provider retained a non-JSON incomplete observation.", + ), + ); } const report = electronPageInspectionSchema.safeParse(result.value); if (!report.success) diff --git a/src/application/javascript/JavaScriptRuntimeObservationService.test.ts b/src/application/javascript/JavaScriptRuntimeObservationService.test.ts index b9351db166..6ef97ff4c2 100644 --- a/src/application/javascript/JavaScriptRuntimeObservationService.test.ts +++ b/src/application/javascript/JavaScriptRuntimeObservationService.test.ts @@ -14,6 +14,35 @@ import { listJavaScriptRuntimeTargets, observeJavaScriptRuntime, } from "./JavaScriptRuntimeObservationService.js"; +import { + expectDroppedPartial, + expectKeptPartial, + expectOutputError, + expectOwnerError, + expectPreemptedCancellation, +} from "../ownerObservationExpectation.fixture.js"; + +const observeCancelled = async (partial: unknown) => { + const reason = { requested: "stop before runtime observation completed" }; + const failure = new AnalysisCancelledError("observe_javascript_runtime", { + cause: reason, + partialObservation: partial as never, + }); + const result = await observeJavaScriptRuntime( + { + identity: () => ({ id: "test", name: "Test", version: "1" }), + listTargets: async () => + err(new AnalysisCancelledError("list_javascript_runtime_targets")), + observe: async () => err(failure), + }, + { + inspector_endpoint: "http://127.0.0.1:9229", + target_id: "target", + observation_ms: 0, + }, + ); + return { result, failure, reason }; +}; it("preserves failed Inspector cleanup when cancellation occurs during observation", async () => { const controller = new AbortController(); @@ -126,11 +155,7 @@ it("does not let late cancellation hide an invalid completed Inspector report", }, { inspector_endpoint: "http://127.0.0.1:9229" }, ); - expect(output).toMatchObject({ - ok: false, - error: { _tag: "AnalysisOutputError" }, - }); - if (!output.ok) expect(output.error).toBeInstanceOf(AnalysisOutputError); + expectOutputError(output); const controller = new AbortController(); const late = await listJavaScriptRuntimeTargets( @@ -147,11 +172,7 @@ it("does not let late cancellation hide an invalid completed Inspector report", { inspector_endpoint: "http://127.0.0.1:9229" }, { signal: controller.signal }, ); - expect(late).toMatchObject({ - ok: false, - error: { _tag: "AnalysisOutputError" }, - }); - if (!late.ok) expect(late.error).toBeInstanceOf(AnalysisOutputError); + expectOutputError(late); }); it("converts a raw runtime observation retained by cleanup failure", async () => { @@ -309,11 +330,23 @@ it.each(["list", "observe"] as const)( { signal: controller.signal }, ); - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.error).toBeInstanceOf(AnalysisCancelledError); - expect(result.error.cause).toBe(reason); - } - expect(providerCalls).toBe(0); + expectPreemptedCancellation(result, reason, providerCalls); + }, +); + +it("keeps cancellation when the retained runtime observation is incomplete", async () => { + const partial = { runtime: "partial", scripts: [] }; + const { result, failure, reason } = await observeCancelled(partial); + expectKeptPartial(expectOwnerError(result, failure, reason), partial); +}); + +it.each([{ scripts: [undefined] }, { elapsed_ms: Number.NaN }])( + "drops a non-JSON runtime partial and keeps cancellation primary", + async (partial) => { + const { result, failure, reason } = await observeCancelled(partial); + expectDroppedPartial( + expectOwnerError(result, failure, reason), + "Inspector retained a non-JSON incomplete observation.", + ); }, ); diff --git a/src/application/javascript/JavaScriptRuntimeObservationService.ts b/src/application/javascript/JavaScriptRuntimeObservationService.ts index 398995f139..0eedaff93b 100644 --- a/src/application/javascript/JavaScriptRuntimeObservationService.ts +++ b/src/application/javascript/JavaScriptRuntimeObservationService.ts @@ -13,7 +13,10 @@ import type { ObserveJavaScriptRuntimeInput, } from "../../domain/javascript/javascriptRuntimeObservation.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { completeProviderEvidence } from "../ProviderOperationExecution.js"; +import { + completeProviderEvidence, + keepIncompleteObservation, +} from "../ProviderOperationExecution.js"; import { javascriptRuntimeObservationSchema, javascriptRuntimeTargetListSchema, @@ -76,15 +79,7 @@ export const observeJavaScriptRuntime = async ( !("kind" in partial) ) { const report = javascriptRuntimeObservationSchema.safeParse(partial); - if (!report.success && result.error instanceof AnalysisCancelledError) - return err( - new AnalysisOutputError( - "observe_javascript_runtime", - "Inspector retained a malformed completed runtime observation.", - { cause: report.error }, - ), - ); - if (report.success) + if (report.success) { result.error.partialObservation = createJavaScriptRuntimeObservationEvidence( "observe_javascript_runtime", @@ -92,8 +87,16 @@ export const observeJavaScriptRuntime = async ( report.data, ready.value.identity(), ); + return result; + } } - return result; + return err( + keepIncompleteObservation( + result.error, + "observe_javascript_runtime", + "Inspector retained a non-JSON incomplete observation.", + ), + ); } const report = javascriptRuntimeObservationSchema.safeParse(result.value); if (!report.success) diff --git a/src/application/ownerObservationExpectation.fixture.ts b/src/application/ownerObservationExpectation.fixture.ts new file mode 100644 index 0000000000..609edb0f48 --- /dev/null +++ b/src/application/ownerObservationExpectation.fixture.ts @@ -0,0 +1,72 @@ +import { expect } from "vitest"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { Result } from "../domain/result.js"; + +/** Assert the service returned the same owner error and its cause. */ +export const expectOwnerError = ( + result: Result, + failure: AnalysisError, + cause: unknown, +): AnalysisError => { + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected the owner failure"); + expect(result.error).toBe(failure); + expect(result.error.cause).toBe(cause); + return result.error; +}; + +/** A JSON-safe incomplete observation stays on the owner error. */ +export const expectKeptPartial = ( + error: AnalysisError, + partial: unknown, + code: "cancelled" | "provider_timeout" = "cancelled", +): void => { + expect(error.partialObservation).toEqual(partial); + expect(error.secondaryFailures).toHaveLength(0); + expect(projectAnalysisError(error)).toMatchObject({ + code, + details: { partial_observation: partial }, + }); +}; + +/** A completed value failed output validation, including after a late abort. */ +export const expectOutputError = ( + result: Result, +): void => { + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); + if (!result.ok) expect(result.error).toBeInstanceOf(AnalysisOutputError); +}; + +/** Cancellation won before the provider ran. */ +export const expectPreemptedCancellation = ( + result: Result, + cause: unknown, + providerCalls: number, +): void => { + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(cause); + expect(providerCalls).toBe(0); +}; + +/** A non-JSON partial is cleared and recorded as a secondary output failure. */ +export const expectDroppedPartial = ( + error: AnalysisError, + reason: string, + code: "cancelled" | "provider_timeout" = "cancelled", +): void => { + expect(error.partialObservation).toBeUndefined(); + expect(error.secondaryFailures).toMatchObject([ + { _tag: "AnalysisOutputError", reason }, + ]); + expect(projectAnalysisError(error).code).toBe(code); +}; diff --git a/src/domain/javascript/electronPageInspection.fixture.ts b/src/domain/javascript/electronPageInspection.fixture.ts new file mode 100644 index 0000000000..e5fe9d0dcb --- /dev/null +++ b/src/domain/javascript/electronPageInspection.fixture.ts @@ -0,0 +1,34 @@ +import { classifyBrowserCompleteness } from "../browserCompleteness.js"; + +/** Empty coverage tail shared by Electron page fixtures. */ +export const emptyElectronPageCoverage = (observationMs = 0) => ({ + capture_window: { + started_at: "2026-01-01T00:00:00.000Z", + ended_at: new Date( + Date.parse("2026-01-01T00:00:00.000Z") + observationMs, + ).toISOString(), + observation_ms: observationMs, + }, + completeness: classifyBrowserCompleteness({ + policyFilteredSections: new Set(), + attachLimitedSections: new Set(), + truncatedSections: new Set(), + unavailableSections: new Set(), + excluded: [], + droppedEvents: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + }, + }), + frames: [], + dom: { total_nodes: 0, nodes: [] }, + scripts: { total: 0, items: [] }, + resources: [], + workers: [], + limitations: [], +}); diff --git a/tests/boundary/mcp/electronMcp.test.ts b/tests/boundary/mcp/electronMcp.test.ts index 9842407fa0..1d6a8fb318 100644 --- a/tests/boundary/mcp/electronMcp.test.ts +++ b/tests/boundary/mcp/electronMcp.test.ts @@ -29,7 +29,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { AsarArtifactReader } from "../../../src/artifacts/AsarArtifactReader.js"; import { toolAvailabilityInventory } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; -import { classifyBrowserCompleteness } from "../../../src/domain/browserCompleteness.js"; +import { emptyElectronPageCoverage } from "../../../src/domain/javascript/electronPageInspection.fixture.js"; import type { ElectronPageInspection } from "../../../src/domain/javascript/electronObservation.js"; import { err } from "../../../src/domain/result.js"; @@ -115,33 +115,7 @@ it("records completed Electron facts after final detach as server Evidence", asy file_path: "/tmp/index.html", attached: true, }, - capture_window: { - started_at: "2026-01-01T00:00:00.000Z", - ended_at: "2026-01-01T00:00:00.001Z", - observation_ms: 1, - }, - completeness: classifyBrowserCompleteness({ - policyFilteredSections: new Set(), - attachLimitedSections: new Set(), - truncatedSections: new Set(), - unavailableSections: new Set(), - excluded: [], - droppedEvents: { - scripts: 0, - network_requests: 0, - console_events: 0, - websocket_connections: 0, - websocket_frames: 0, - webmcp_tools: 0, - timeline_events: 0, - }, - }), - frames: [], - dom: { total_nodes: 0, nodes: [] }, - scripts: { total: 0, items: [] }, - resources: [], - workers: [], - limitations: [], + ...emptyElectronPageCoverage(1), }; const provider: ElectronObservationPort = { identity: () => ({ From 6a6adf703fa44100643947d20fbc670f86260b1b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:21:16 +0800 Subject: [PATCH 11/35] fix(resources)!: inspect app resources without an executable Add a read-only app resource session shared by CLI and MCP. Preserve resource-root provenance without inventing executable identity, reject binary-only snapshots and operations, and retain failed cleanup owners across switches and retries. Keep filesystem admission failures typed and preserve cancellation ordering from observed cleanup facts. Document resource routing in the packaged investigation skill. BREAKING CHANGE: resource-only observations may have a null Evidence subject and null executable target digests. Clients must use resource_root and reported per-resource provenance instead. Refs #5589 --- .../skills/reverse-engineer-anything/SKILL.md | 4 + .../references/native-and-artifacts.md | 16 + docs/mcp-contracts.md | 11 + src/application/AnalysisProvider.ts | 48 +- src/application/AppBundleExecutable.ts | 58 ++ src/application/AppBundleResourceRoot.test.ts | 101 +++ src/application/DirectAnalysis.ts | 46 +- .../DirectAnalysisStatFailure.test.ts | 117 +++ src/application/FunctionAnalysisEvidence.ts | 34 +- .../android/AndroidAnalysisService.ts | 26 +- .../binary/AnalysisSnapshotCache.test.ts | 78 ++ .../binary/AnalysisSnapshotCache.ts | 21 +- src/application/binary/BinarySession.ts | 431 +++++++++- .../BinarySessionBundleResources.test.ts | 799 ++++++++++++++++++ .../binary/BinarySessionComposition.ts | 3 + .../binary/BinarySessionExecution.ts | 5 +- .../binary/BinarySessionLiveness.test.ts | 39 + src/application/binary/BinarySessionPort.ts | 6 + src/application/binary/BinarySessionStatus.ts | 73 +- .../binary/BundleResourceSession.test.ts | 106 +++ .../binary/BundleResourceSession.ts | 85 ++ .../firmware/FirmwareAnalysisService.ts | 2 +- .../javascript/JavaScriptRecoveryService.ts | 2 +- src/application/native/NativeValueTrace.ts | 2 +- .../sqlite/SqliteDatabaseService.ts | 2 +- src/artifacts/ArtifactProvider.ts | 121 ++- .../BundleResourceArtifactClient.test.ts | 60 ++ .../apple/AppleAssetCatalogAnalysis.ts | 9 +- .../apple/InterfaceBuilderAnalysis.ts | 6 +- src/artifacts/apple/KeyedArchiveReader.ts | 6 +- src/cliBundleResource.test.ts | 69 ++ src/composition/binary.ts | 73 +- src/contracts/sessionLifecycleInputs.ts | 5 + src/contracts/sessionToolContracts.ts | 10 +- src/contracts/toolContractExamples.ts | 1 + src/contracts/toolEffects.ts | 1 + src/contracts/toolOutputSchemaGroups.ts | 7 + src/domain/analysisView/analysisView.ts | 7 +- .../analysisView/interfaceBuilderView.test.ts | 44 + .../analysisView/interfaceBuilderView.ts | 23 +- src/domain/apple/appleAssetCatalog.test.ts | 28 + src/domain/apple/appleAssetCatalog.ts | 33 +- src/domain/apple/bundleResourceRoot.ts | 33 + .../apple/interfaceBuilderGraph.test.ts | 27 + src/domain/apple/interfaceBuilderGraph.ts | 25 +- src/domain/apple/keyedArchive.ts | 70 +- src/domain/native/nativeInspection.ts | 2 + src/domain/native/nativeInvestigationGraph.ts | 24 +- src/domain/pcap/pcapEvidence.ts | 2 +- src/native/NativeMacOSProvider.ts | 67 +- src/native/NativePlistResourceRoot.test.ts | 46 + src/native/NativePlistSource.ts | 16 + src/providers/AnalysisClientCleanup.ts | 26 +- src/server/BundleResourcesMcp.test.ts | 428 ++++++++++ src/server/registerEvidenceTools.ts | 4 +- src/server/registerSessionTools.ts | 34 + .../bundleResourceRootCliMcp.test.ts | 234 +++++ .../bundleResourceComposition.test.ts | 115 +++ 58 files changed, 3582 insertions(+), 189 deletions(-) create mode 100644 src/application/AppBundleResourceRoot.test.ts create mode 100644 src/application/DirectAnalysisStatFailure.test.ts create mode 100644 src/application/binary/BinarySessionBundleResources.test.ts create mode 100644 src/application/binary/BundleResourceSession.test.ts create mode 100644 src/application/binary/BundleResourceSession.ts create mode 100644 src/artifacts/BundleResourceArtifactClient.test.ts create mode 100644 src/cliBundleResource.test.ts create mode 100644 src/domain/apple/bundleResourceRoot.ts create mode 100644 src/native/NativePlistResourceRoot.test.ts create mode 100644 src/server/BundleResourcesMcp.test.ts create mode 100644 tests/acceptance/applications/bundleResourceRootCliMcp.test.ts create mode 100644 tests/boundary/providers/bundleResourceComposition.test.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 510ddb97fc..e283e35b62 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -94,6 +94,10 @@ explicit path or endpoint and do not need it. - Archive/package member inventory (ZIP/APK/IPA/MSIX/AppX or DMG): `open_binary` with the supplied local path; use `inspect_artifact` when its graph and findings help answer the question. +- Apple app plist, Interface Builder, asset catalog, or keyed archive resources: + `open_bundle_resources`, then the selected resource tool. This does not + require `CFBundleExecutable` or a deep provider. Resource sessions have no + executable digest or binary snapshot; use `open_binary` for native code. - Android APK code, classes, methods, or incoming references: `inspect_android_package`, then focused Android tools when advertised. Archive member inventory still uses the archive route above. diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 122e510a99..896378dc9a 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -11,6 +11,22 @@ Select columns, batch IDs and page individual facts; aliases remain unresolved. Unpaired observations do not prove additions/removals. Existing matched dossier pairs include inputs for `compare_functions`; otherwise analyze just chosen pairs. +## Apple app resources + +Use `open_bundle_resources` with the selected app path, then `inspect_plist`, +`decode_interface_builder`, `inspect_asset_catalog`, or `inspect_keyed_archive`. +The resource session reads the app without resolving `CFBundleExecutable` or +starting a deep provider. Preserve `resource_root` provenance, per-resource +hashes where reported, and null executable identity; a directory path is not +an executable digest. Export retained Evidence before closing or switching +roots. Binary analysis snapshots do not apply. Use `open_binary` to switch to +executable analysis, and `close_binary` to release either session kind. + +The four matching CLI commands accept the app path directly. For an archive +inside the app, use `inspect-keyed-archive APP_PATH RELATIVE_ARCHIVE_PATH`. +Host requirements for plist and compiled asset/Interface Builder tooling still +apply; a resource-only session does not establish provider availability. + ## Desktop installation directories When an installation directory is supplied and its runtime is unknown, use diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 0168c51ad8..845e8c8719 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -528,6 +528,17 @@ concrete version, selection source, and complete analysis profile through choices return typed selection details. A selected provider is never replaced automatically after a runtime failure. +`open_bundle_resources` selects a separate read-only app resource session +without resolving `CFBundleExecutable`, including when it is absent or +unresolvable. `binary_session` reports `session_kind: "bundle_resources"` and the +caller and canonical resource-root paths; it does not report a binary target or +deep-provider binding. `inspect_plist`, `decode_interface_builder`, +`inspect_asset_catalog`, and `inspect_keyed_archive` retain resource-root +provenance and use a null Evidence subject when executable identity is unknown. +Binary-only operations and binary analysis snapshots do not apply in this +session. `open_binary` can switch back to executable analysis, and +`close_binary` retires either session kind. + Each provider client receives `binary_session.analysis_run.run_id` before any provider startup, including clients whose startup fails. `process_lineage` is `not_observed` until a dynamic provider starts, then becomes `snapshots` with diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 70f5cc464e..70d61ddd4b 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -68,7 +68,11 @@ export interface AnalysisExecution { */ readonly referencesTruncated?: boolean; readonly locations: readonly EvidenceLocation[]; - readonly subject: EvidenceSubjectTarget | null; + /** + * Missing means the selected operation's default subject should be bound by + * its caller. Explicit null means the observation has no file subject. + */ + readonly subject?: EvidenceSubjectTarget | null; } interface CreateAnalysisExecutionOptions { @@ -79,7 +83,7 @@ interface CreateAnalysisExecutionOptions { readonly relationshipCoverage?: RelationshipCoverage; readonly referencesTruncated?: boolean; readonly locations?: readonly EvidenceLocation[]; - readonly subject?: EvidenceSubjectTarget; + readonly subject?: EvidenceSubjectTarget | null; } /** @@ -129,7 +133,7 @@ export function createAnalysisExecution( ? {} : { referencesTruncated: options.referencesTruncated }), locations: [...(options.locations ?? [])], - subject: options.subject ?? null, + ...(options.subject === undefined ? {} : { subject: options.subject }), }; } @@ -143,21 +147,27 @@ export const createAnalysisExecutionEvidence = ( execution: AnalysisExecution, confidence: "observed" | "derived", ): Evidence => - createEvidence(execution.subject ?? undefined, execution.provider, { - operation, - parameters: jsonObjectSchema.parse(input), - result: execution.result, - rawResult: execution.rawResult, - limitations: execution.limitations, - ...(execution.analysisProfile === undefined - ? {} - : { analysisProfile: execution.analysisProfile }), - ...(execution.relationshipCoverage === undefined - ? {} - : { relationshipCoverage: execution.relationshipCoverage }), - locations: execution.locations, - confidence, - }); + createEvidence( + execution.subject === undefined + ? undefined + : (execution.subject ?? undefined), + execution.provider, + { + operation, + parameters: jsonObjectSchema.parse(input), + result: execution.result, + rawResult: execution.rawResult, + limitations: execution.limitations, + ...(execution.analysisProfile === undefined + ? {} + : { analysisProfile: execution.analysisProfile }), + ...(execution.relationshipCoverage === undefined + ? {} + : { relationshipCoverage: execution.relationshipCoverage }), + locations: execution.locations, + confidence, + }, + ); /** Provider-neutral application capability for one analysis operation. */ export interface AnalysisOperationPort { @@ -206,6 +216,8 @@ export interface AnalysisClient extends AnalysisOperationPort { close( options?: Pick & { readonly retainDocument?: boolean; + /** Report each owned cleanup failure as soon as its owner observes it. */ + readonly onCleanupFailure?: (failure: AnalysisError) => void; }, ): Promise>; } diff --git a/src/application/AppBundleExecutable.ts b/src/application/AppBundleExecutable.ts index eff93d5dae..eb160ead28 100644 --- a/src/application/AppBundleExecutable.ts +++ b/src/application/AppBundleExecutable.ts @@ -14,6 +14,7 @@ import { import { decodeXmlPlistText } from "../domain/propertyListXmlText.js"; import { hasBinaryPlistSignature } from "../domain/apple/appleArchiveFormat.js"; import { err, ok, type Result } from "../domain/result.js"; +import { absoluteCallerPath } from "../filesystem/callerPath.js"; /** Decoded representation bound for one binary Info.plist. */ const BINARY_PLIST_DECODE_BYTES = 16 * 1024 * 1024; @@ -42,6 +43,18 @@ export interface ResolvedAppBundle { readonly infoPlist?: string; } +/** Resource-only identity for an app directory; it is not a BinaryTarget. */ +export interface ResolvedAppBundleResourceRoot { + /** Path selected by the caller, retained even when it is a symlink alias. */ + readonly callerPath: string; + /** Canonical directory whose resources are inspected. */ + readonly canonicalPath: string; + readonly infoPlistPath: string; + readonly layout: "contents" | "root"; + /** No executable was opened or hashed by resource-root discovery. */ + readonly executableSha256: string | null; +} + /** Filesystem operations used by the resolver, injectable for boundary tests. */ export interface AppBundleFileSystem { readonly readFile: (path: string) => Promise; @@ -90,6 +103,51 @@ export const resolveAppBundleExecutable = async ( ); }; +/** + * Resolve only the root and Info.plist layout needed for read-only resource + * inspection. This intentionally does not require or guess CFBundleExecutable. + */ +export const resolveAppBundleResourceRoot = async ( + path: string, + options: { readonly cwd?: string } = {}, +): Promise> => { + const callerPath = absoluteCallerPath( + path, + options.cwd === undefined ? {} : { cwd: options.cwd }, + ); + try { + const canonicalInput = await realpath(callerPath); + const directLayout = await presentLayout(canonicalInput); + const wrapped = + directLayout === undefined + ? await wrappedBundle(canonicalInput) + : undefined; + const root = wrapped ?? canonicalInput; + const layout = directLayout ?? (await presentLayout(root)); + if (layout === undefined) + return err( + new BinaryTargetError( + callerPath, + `app resource root has no supported Info.plist layout: ${callerPath}`, + ), + ); + const canonicalPath = await realpath(root); + return ok({ + callerPath, + canonicalPath, + infoPlistPath: join(canonicalPath, ...layout.plist), + layout: layout.plist[0] === "Contents" ? "contents" : "root", + executableSha256: null, + }); + } catch (cause: unknown) { + return err( + new BinaryTargetError(callerPath, errorReason(cause, callerPath), { + cause, + }), + ); + } +}; + const presentLayout = async ( bundle: string, ): Promise => { diff --git a/src/application/AppBundleResourceRoot.test.ts b/src/application/AppBundleResourceRoot.test.ts new file mode 100644 index 0000000000..9b8a53da5b --- /dev/null +++ b/src/application/AppBundleResourceRoot.test.ts @@ -0,0 +1,101 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { + createAnalysisExecution, + createAnalysisExecutionEvidence, +} from "./AnalysisProvider.js"; +import { resolveAppBundleResourceRoot } from "./AppBundleExecutable.js"; +import { bindExecutionTarget } from "./binary/BinarySessionExecution.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; + +const provider = { id: "test", name: "test", version: null } as const; +const target = { + path: "/Fixture.app/Contents/Info.plist", + sha256: "a".repeat(64), + kind: "artifact", + format: "plist", +} satisfies BinaryTarget; + +it("resolves resource roots without requiring or hashing CFBundleExecutable", async () => { + const temp = await mkdtemp(join(tmpdir(), "rea-app-resource-root-")); + try { + const bundle = join(temp, "Fixture.app"); + const contents = join(bundle, "Contents"); + const alias = join(temp, "Selected.app"); + await mkdir(contents, { recursive: true }); + await writeFile(join(contents, "Info.plist"), ""); + await symlink(bundle, alias); + + const resolved = await resolveAppBundleResourceRoot(alias); + + expect(resolved).toEqual({ + ok: true, + value: { + callerPath: alias, + canonicalPath: bundle, + infoPlistPath: join(bundle, "Contents", "Info.plist"), + layout: "contents", + executableSha256: null, + }, + }); + } finally { + await rm(temp, { recursive: true, force: true }); + } +}); + +it("resolves a flat bundle resource root without inferring executable identity", async () => { + const temp = await mkdtemp(join(tmpdir(), "rea-ios-resource-root-")); + try { + const bundle = join(temp, "Fixture.app"); + await mkdir(bundle); + await writeFile(join(bundle, "Info.plist"), ""); + + const resolved = await resolveAppBundleResourceRoot(bundle); + + expect(resolved).toMatchObject({ + ok: true, + value: { + callerPath: bundle, + canonicalPath: bundle, + infoPlistPath: join(bundle, "Info.plist"), + layout: "root", + executableSha256: null, + }, + }); + } finally { + await rm(temp, { recursive: true, force: true }); + } +}); + +it("distinguishes default target binding from an explicitly unbound observation", () => { + const defaultSubject = createAnalysisExecution({ ok: true }, provider); + const unbound = createAnalysisExecution({ ok: true }, provider, { + subject: null, + }); + + const bound = bindExecutionTarget( + { ok: true, value: defaultSubject }, + "inspect_plist", + target, + ); + const preserved = bindExecutionTarget( + { ok: true, value: unbound }, + "inspect_plist", + target, + ); + + expect(bound.ok && bound.value.subject).toEqual({ + path: target.path, + sha256: target.sha256, + format: "plist", + }); + expect(preserved.ok && preserved.value.subject).toBeNull(); + expect( + createAnalysisExecutionEvidence("inspect_plist", {}, unbound, "observed") + .subject, + ).toBeNull(); +}); diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index 157309c616..d6255030ba 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -24,10 +24,14 @@ import { } from "../domain/evidenceErrors.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; -import { access } from "node:fs/promises"; +import { access, stat } from "node:fs/promises"; +import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { readAnalysisSnapshot } from "./binary/AnalysisSnapshotFiles.js"; import { requiresNativeSourceAdmissionBeforeReplay } from "./binary/AnalysisSnapshotCache.js"; -import { parseBinaryTarget } from "./BinaryTargetResolver.js"; +import { + classifyBinaryTargetFilesystemFailure, + parseBinaryTarget, +} from "./BinaryTargetResolver.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { snapshotEvidenceForQuery, @@ -56,11 +60,13 @@ import type { AnalysisProviderSelector } from "../contracts/providerSelection.js import { artifactInspectionResultSchema } from "../domain/artifactInspection.js"; import { resolveXrefsAddress } from "./XrefsAddressResolution.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import type { BinaryTargetError } from "../domain/configurationErrors.js"; import { directAnalysisCleanupFailure, managedAnalysisCleanupFailure, withSessionCleanup, } from "./DirectAnalysisCleanup.js"; +import { BUNDLE_RESOURCE_OPERATIONS } from "./binary/BundleResourceSession.js"; type DirectAnalysisTool = | "annotate_native_function" @@ -97,7 +103,6 @@ const NATIVE_INSPECTION_OPERATIONS: ReadonlySet = new Set([ const ARTIFACT_ANALYSIS_OPERATION_SET: ReadonlySet = new Set( ARTIFACT_ANALYSIS_OPERATIONS, ); - /** * Open one binary, execute one tool, and always release provider resources. * Unlike MCP mode, every CLI invocation is intentionally isolated and does not @@ -249,6 +254,24 @@ const runAnalysis = async ( return withSessionCleanup( session, async () => { + let isDirectory = false; + if (BUNDLE_RESOURCE_OPERATIONS.has(tool)) { + const inspected = await inspectBundleResourceInput(path); + if (!inspected.ok) return cliError(inspected.error); + isDirectory = inspected.value; + } + if (BUNDLE_RESOURCE_OPERATIONS.has(tool) && isDirectory) { + const opened = await session.openBundleResources(path, { signal }); + if (!opened.ok) return cliError(opened.error); + const execution = await session.execute(tool, arguments_, { signal }); + if (!execution.ok) return cliError(execution.error); + return createAnalysisExecutionEvidence( + tool, + arguments_, + execution.value, + "observed", + ); + } const prepared = await prepareSnapshot({ path, snapshotPath, @@ -493,7 +516,9 @@ const executeAnalysisTool = async (input: { } if (!result.ok) return { output: cliError(result.error) }; const evidence = createEvidence( - result.value.subject ?? input.openedTarget, + result.value.subject === undefined + ? input.openedTarget + : (result.value.subject ?? undefined), result.value.provider, { operation: tool, @@ -577,6 +602,19 @@ const prepareSnapshot = async (options: { return ok({ snapshot: loaded.value, target: target.value }); }; +const inspectBundleResourceInput = async ( + path: string, +): Promise> => { + const callerPath = absoluteCallerPath(path); + try { + return ok((await stat(callerPath)).isDirectory()); + } catch (cause: unknown) { + const failure = classifyBinaryTargetFilesystemFailure(callerPath, cause); + if (failure !== undefined) return err(failure); + throw cause; + } +}; + const allowsSnapshotReplay = ( route: SessionProviderRoute, tool: diff --git a/src/application/DirectAnalysisStatFailure.test.ts b/src/application/DirectAnalysisStatFailure.test.ts new file mode 100644 index 0000000000..0d82f7b493 --- /dev/null +++ b/src/application/DirectAnalysisStatFailure.test.ts @@ -0,0 +1,117 @@ +import { afterEach, expect, it, vi } from "vitest"; + +const statFailures = vi.hoisted(() => new Map()); + +vi.mock("node:fs/promises", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + stat: vi.fn(async (...args: Parameters) => { + const failure = statFailures.get(String(args[0])); + if (failure !== undefined) throw failure; + return original.stat(...args); + }), + }; +}); + +import { parseConfig } from "../config/parseConfig.js"; +import { classifyBinaryTargetFilesystemFailure } from "./BinaryTargetResolver.js"; +import type { BinarySession } from "./binary/BinarySession.js"; +import type { DirectAnalysisDependencies } from "./DirectAnalysisDependencies.js"; +import { runProviderAnalysis } from "./DirectAnalysis.js"; + +afterEach(() => statFailures.clear()); + +it("retains the original filesystem cause on the shared typed error", () => { + const cause = Object.assign(new Error("permission denied"), { + code: "EACCES", + }); + const failure = classifyBinaryTargetFilesystemFailure( + "/tmp/rea-stat-failure-cause.app", + cause, + ); + + expect(failure?.cause).toBe(cause); + expect(failure?.systemCode).toBe("EACCES"); +}); + +it.each([ + { code: "ENOENT", projectedCode: "target_unavailable" }, + { code: "ENOTDIR", projectedCode: "target_unavailable" }, + { code: "EACCES", projectedCode: "access_denied", systemCode: "EACCES" }, + { code: "EPERM", projectedCode: "access_denied", systemCode: "EPERM" }, + { code: "EIO", projectedCode: "target_unavailable" }, +])( + "stops CLI resource dispatch on stat failure $code and preserves its path", + async ({ code, projectedCode, systemCode }) => { + const path = `/tmp/rea-stat-failure-${code}.app`; + const cause = Object.assign(new Error(`${code}: injected stat failure`), { + code, + }); + statFailures.set(path, cause); + const session = { + close: vi.fn(async () => ({ ok: true as const, value: null })), + open: vi.fn(), + openBundleResources: vi.fn(), + execute: vi.fn(), + } as unknown as BinarySession; + const dependencies: DirectAnalysisDependencies = { + readConfiguration: () => parseConfig({}), + createBinarySession: () => session, + createNativeDispatchBinarySession: () => session, + createArtifactBinarySession: () => session, + createNativeBinarySession: () => session, + createManagedBinarySession: () => session, + }; + + const output = await runProviderAnalysis( + dependencies, + path, + "inspect_keyed_archive", + { path: "Resources/Archive.plist" }, + ); + + expect(output).toMatchObject({ + code: projectedCode, + details: { + path, + reason: expect.stringContaining(cause.message), + ...(systemCode === undefined ? {} : { system_code: systemCode }), + }, + }); + expect(session.open).not.toHaveBeenCalled(); + expect(session.openBundleResources).not.toHaveBeenCalled(); + expect(session.execute).not.toHaveBeenCalled(); + expect(session.close).toHaveBeenCalledOnce(); + }, +); + +it("rethrows an unclassified stat failure instead of treating it as a file", async () => { + const path = "/tmp/rea-stat-failure-opaque.app"; + const cause = new Error("opaque filesystem failure"); + statFailures.set(path, cause); + const session = { + close: vi.fn(async () => ({ ok: true as const, value: null })), + open: vi.fn(), + openBundleResources: vi.fn(), + execute: vi.fn(), + } as unknown as BinarySession; + const dependencies: DirectAnalysisDependencies = { + readConfiguration: () => parseConfig({}), + createBinarySession: () => session, + createNativeDispatchBinarySession: () => session, + createArtifactBinarySession: () => session, + createNativeBinarySession: () => session, + createManagedBinarySession: () => session, + }; + + await expect( + runProviderAnalysis(dependencies, path, "inspect_keyed_archive", { + path: "Resources/Archive.plist", + }), + ).rejects.toBe(cause); + expect(session.open).not.toHaveBeenCalled(); + expect(session.openBundleResources).not.toHaveBeenCalled(); + expect(session.execute).not.toHaveBeenCalled(); + expect(session.close).toHaveBeenCalledOnce(); +}); diff --git a/src/application/FunctionAnalysisEvidence.ts b/src/application/FunctionAnalysisEvidence.ts index 00ef766a8f..9042082137 100644 --- a/src/application/FunctionAnalysisEvidence.ts +++ b/src/application/FunctionAnalysisEvidence.ts @@ -39,19 +39,25 @@ export const executeFunctionAnalysisEvidence = async ( if (!dossier.ok) return dossier; const observation = execution.value; return ok( - createEvidence(observation.subject ?? target, observation.provider, { - operation: "analyze_function", - parameters: arguments_, - result: dossier.value, - rawResult: observation.rawResult, - limitations: observation.limitations, - locations: observation.locations, - ...(observation.relationshipCoverage === undefined - ? {} - : { relationshipCoverage: observation.relationshipCoverage }), - ...(observation.analysisProfile === undefined - ? {} - : { analysisProfile: observation.analysisProfile }), - }), + createEvidence( + observation.subject === undefined + ? target + : (observation.subject ?? undefined), + observation.provider, + { + operation: "analyze_function", + parameters: arguments_, + result: dossier.value, + rawResult: observation.rawResult, + limitations: observation.limitations, + locations: observation.locations, + ...(observation.relationshipCoverage === undefined + ? {} + : { relationshipCoverage: observation.relationshipCoverage }), + ...(observation.analysisProfile === undefined + ? {} + : { analysisProfile: observation.analysisProfile }), + }, + ), ); }; diff --git a/src/application/android/AndroidAnalysisService.ts b/src/application/android/AndroidAnalysisService.ts index 45c802dc96..4e2e5841d4 100644 --- a/src/application/android/AndroidAnalysisService.ts +++ b/src/application/android/AndroidAnalysisService.ts @@ -45,16 +45,22 @@ export class AndroidAnalysisService { if (!executed.ok) return executed; const execution = executed.value; return completeProviderEvidence( - createEvidence(execution.subject ?? target.value, execution.provider, { - operation, - parameters: jsonObjectSchema.parse(request.data.input), - result: execution.result, - rawResult: execution.rawResult, - limitations: execution.limitations, - locations: execution.locations, - confidence: - operation === "inspect_android_method" ? "derived" : "observed", - }), + createEvidence( + execution.subject === undefined + ? target.value + : (execution.subject ?? undefined), + execution.provider, + { + operation, + parameters: jsonObjectSchema.parse(request.data.input), + result: execution.result, + rawResult: execution.rawResult, + limitations: execution.limitations, + locations: execution.locations, + confidence: + operation === "inspect_android_method" ? "derived" : "observed", + }, + ), options?.signal, ); } diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 215d1b1dde..df77808dc6 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -175,6 +175,84 @@ it("rebinds replayed target subjects while preserving child artifact subjects", ).toMatchObject(childSubject); }); +it("binds an omitted subject to the selected target in snapshot replay", () => { + const cache = new AnalysisSnapshotCache(); + const target = ANALYSIS_SNAPSHOT_TARGET; + const profile = ANALYSIS_SNAPSHOT_PROFILE; + const relocatedTarget = { ...target, path: "/tmp/relocated-default-target" }; + const parameters = { address: "0x2a00" }; + const execution = createAnalysisExecution( + { name: "default-bound" }, + ANALYSIS_SNAPSHOT_PROVIDER, + { analysisProfile: profile }, + ); + expect(execution).not.toHaveProperty("subject"); + cache.record({ + target, + profile, + operation: "procedure_info", + parameters, + execution, + }); + + const snapshot = cache.export( + target, + profile, + createEvidenceBundle([ + createEvidence(target, ANALYSIS_SNAPSHOT_PROVIDER, { + operation: "procedure_info", + parameters, + result: { name: "default-bound" }, + analysisProfile: profile, + }), + ]), + ); + if (!snapshot.ok) throw snapshot.error; + expect(snapshot.value.entries[0]?.execution.subject?.path).toBe(target.path); + const reparsed = parseAnalysisSnapshot( + JSON.parse(serializeAnalysisSnapshot(snapshot.value)), + ); + expect(reparsed.entries[0]?.execution.subject?.path).toBe(target.path); + cache.clear(); + const imported = cache.import( + reparsed, + { target: relocatedTarget, profile }, + () => ok(0), + ); + expect(imported.ok).toBe(true); + expect( + cache.lookup(relocatedTarget, profile, "procedure_info", parameters) + ?.subject?.path, + ).toBe(relocatedTarget.path); +}); + +it("preserves an explicitly unbound subject through snapshot replay", () => { + const cache = new AnalysisSnapshotCache(); + const target = ANALYSIS_SNAPSHOT_TARGET; + const profile = ANALYSIS_SNAPSHOT_PROFILE; + const parameters = { address: "0x3000" }; + cache.record({ + target, + profile, + operation: "procedure_info", + parameters, + execution: createAnalysisExecution( + { name: "unbound" }, + ANALYSIS_SNAPSHOT_PROVIDER, + { subject: null, analysisProfile: profile }, + ), + }); + + expect( + cache.lookup( + { ...target, path: "/tmp/relocated-unbound-target" }, + profile, + "procedure_info", + parameters, + )?.subject, + ).toBeNull(); +}); + describe("analysis snapshot cache capacity", () => { it("shares capacity across query kinds while allowing replacements and fresh partitions", () => { const cache = new AnalysisSnapshotCache(); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index 79e212cac6..cc64030b7d 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -254,7 +254,7 @@ export class AnalysisSnapshotCache { readonly limitations: readonly string[]; readonly relationshipCoverage?: AnalysisExecution["relationshipCoverage"]; readonly locations: AnalysisExecution["locations"]; - readonly subject: AnalysisExecution["subject"]; + readonly subject: NonNullable | null; }; }): void { this.select(input.target, input.profile); @@ -292,7 +292,7 @@ export class AnalysisSnapshotCache { if (cached === undefined) return undefined; const cachedSubject = cached.execution.subject; const subject = - cachedSubject === null + cachedSubject == null ? null : rebindEvidenceSubjectTargetPath( { @@ -328,7 +328,7 @@ export class AnalysisSnapshotCache { relationshipCoverage: cached.execution.relationship_coverage, }), locations: cached.execution.locations, - subject, + subject: subject ?? null, }); } @@ -357,7 +357,20 @@ export class AnalysisSnapshotCache { binding: snapshotBinding(profile), operation, parameters, - execution, + execution: { + ...execution, + subject: + execution.subject === undefined + ? { + path: target.path, + sha256: target.sha256, + format: target.format, + ...(target.architecture === undefined + ? {} + : { architecture: target.architecture }), + } + : execution.subject, + }, }); this.#entries.set(entry.query_id, entry); } diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index 425c530126..ed8e64fbfa 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -7,9 +7,10 @@ import { } from "../../domain/analysisProfile.js"; import { AnalysisCancelledError, + AnalysisCapabilityUnavailableError, AnalysisOutputError, } from "../../domain/analysisErrorCore.js"; -import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../../domain/result.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { createEvidence } from "../../domain/evidence.js"; @@ -48,7 +49,19 @@ import { } from "./BinarySessionExecution.js"; import { closeAnalysisClient } from "../../providers/AnalysisClientCleanup.js"; import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { profilelessNativeDispatchProfile } from "./ProfilelessNativeWorkflowSnapshot.js"; +import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js"; +import { AnalysisSnapshotMismatchError } from "../../domain/evidenceErrors.js"; +import { + resolveAppBundleResourceRoot, + type ResolvedAppBundleResourceRoot, +} from "../AppBundleExecutable.js"; +import { + BUNDLE_RESOURCE_OPERATIONS, + type ActiveBundleResourceSession, + type BundleResourceClientFactory, +} from "./BundleResourceSession.js"; const OFFICIAL_OPERATIONS: ReadonlySet = new Set( OFFICIAL_TOOL_CONTRACTS.map(({ name }) => name), ); @@ -69,6 +82,56 @@ interface SessionBinding extends SessionRun { nativeSourceAdmitted: boolean; } +interface CleanupFailureTracker { + readonly error: AnalysisError | undefined; + readonly onFailure: (failure: AnalysisError) => void; +} + +const trackCleanupFailures = ( + signal: AbortSignal | undefined, + operation: string, +): CleanupFailureTracker => { + let error: AnalysisError | undefined; + let cleanupFailure: AnalysisError | undefined; + let cancellation: AnalysisCancelledError | undefined; + const observed = new Set(); + return { + get error() { + return error; + }, + onFailure: (failure) => { + if ( + observed.has(failure) || + (failure.cleanupPrimary !== undefined && + observed.has(failure.cleanupPrimary)) + ) + return; + observed.add(failure); + if (cleanupFailure === undefined) { + if (isAborted(signal)) + cancellation = new AnalysisCancelledError(operation, { + cause: signal?.reason, + }); + cleanupFailure = failure; + } else { + cleanupFailure = analysisErrorWithCleanupFailure( + cleanupFailure, + failure, + operation, + ); + } + error = + cancellation === undefined + ? cleanupFailure + : analysisErrorWithCleanupFailure( + cancellation, + cleanupFailure, + operation, + ); + }, + }; +}; + /** * Owns the single active target shared by CLI and MCP adapters. * @@ -94,6 +157,9 @@ export class BinarySession readonly #calls = new Map, number>(); readonly #promptCompletionControllers = new Set(); readonly #providerRouter: SessionProviderRouter; + readonly #bundleResourceFactory: BundleResourceClientFactory | undefined; + #activeBundleResources: ActiveBundleResourceSession | undefined; + #pendingBundleResourceCleanup: ActiveBundleResourceSession[] = []; readonly #runtimeUnavailability = new Map< string, { readonly reason: string } @@ -103,9 +169,161 @@ export class BinarySession constructor( providerRouter: SessionProviderRouter, records: InvestigationRecords = new InvestigationRecords(), + bundleResourceFactory?: BundleResourceClientFactory, ) { super(records); this.#providerRouter = providerRouter; + this.#bundleResourceFactory = bundleResourceFactory; + } + + /** Open an app resource root without requiring CFBundleExecutable. */ + openBundleResources( + path: string, + options: Pick = {}, + ): Promise> { + return this.#serialize(async (admittedThrough) => { + await this.#drainCalls(admittedThrough); + if (isAborted(options.signal)) + return err( + new AnalysisCancelledError("open_bundle_resources", { + cause: options.signal?.reason, + }), + ); + if (this.#externalMutationCleanupFailure?.cleanupIncomplete === true) + return err(this.#externalMutationCleanupFailure); + if (this.#bundleResourceFactory === undefined) + return err( + new AnalysisCapabilityUnavailableError( + "bundle-resources", + "open_bundle_resources", + "No app resource reader is configured.", + ), + ); + const resolved = await resolveAppBundleResourceRoot(path); + if (!resolved.ok) return resolved; + if (isAborted(options.signal)) + return err( + new AnalysisCancelledError("open_bundle_resources", { + cause: options.signal?.reason, + }), + ); + const pendingClosed = await this.#retryPendingCleanup(); + if (!pendingClosed.ok) return pendingClosed; + const pendingResourcesClosed = + await this.#retryPendingBundleResourceCleanup(); + if (!pendingResourcesClosed.ok) return pendingResourcesClosed; + if (isAborted(options.signal)) + return err( + new AnalysisCancelledError("open_bundle_resources", { + cause: options.signal?.reason, + }), + ); + const runId = randomUUID(); + let bundleClient: Awaited< + ReturnType + >; + try { + bundleClient = await this.#bundleResourceFactory.createClient( + resolved.value, + ); + } catch (cause: unknown) { + return err( + cause instanceof AnalysisError + ? cause + : new ProviderAdapterError( + "bundle-resources", + "open_bundle_resources", + { cause }, + ), + ); + } + const candidate: ActiveBundleResourceSession = { + root: resolved.value, + client: bundleClient.client, + capabilities: bundleClient.capabilities, + runId, + }; + const started = await candidate.client.execute("health", {}, options); + if (!started.ok) { + this.#pendingBundleResourceCleanup.push(candidate); + const closed = await this.#retryPendingBundleResourceCleanup(); + return err( + closed.ok + ? started.error + : analysisErrorWithCleanupFailure( + started.error, + closed.error, + "open_bundle_resources", + ), + ); + } + if (isAborted(options.signal)) { + const cancelled = new AnalysisCancelledError("open_bundle_resources", { + cause: options.signal?.reason, + }); + this.#pendingBundleResourceCleanup.push(candidate); + const closed = await this.#retryPendingBundleResourceCleanup(); + return err( + closed.ok + ? cancelled + : analysisErrorWithCleanupFailure( + cancelled, + closed.error, + "open_bundle_resources", + ), + ); + } + const cleanupFailures = trackCleanupFailures( + options.signal, + "open_bundle_resources", + ); + const closed = await this.#closeActive( + {}, + false, + cleanupFailures.onFailure, + ); + if (!closed.ok) { + this.#pendingBundleResourceCleanup.push(candidate); + const candidateClosed = await this.#retryPendingBundleResourceCleanup( + undefined, + cleanupFailures.onFailure, + ); + const primary = cleanupFailures.error ?? closed.error; + return err( + cleanupFailures.error ?? + (candidateClosed.ok + ? primary + : analysisErrorWithCleanupFailure( + primary, + candidateClosed.error, + "open_bundle_resources", + )), + ); + } + if (isAborted(options.signal)) { + const cancelled = new AnalysisCancelledError("open_bundle_resources", { + cause: options.signal?.reason, + }); + this.#pendingBundleResourceCleanup.push(candidate); + const candidateClosed = await this.#retryPendingBundleResourceCleanup( + undefined, + cleanupFailures.onFailure, + ); + return err( + cleanupFailures.error ?? + (candidateClosed.ok + ? cancelled + : analysisErrorWithCleanupFailure( + cancelled, + candidateClosed.error, + "open_bundle_resources", + )), + ); + } + this.#activeBundleResources = candidate; + this.resetSnapshotInvalidation(); + return resolved; + }); } /** @@ -123,7 +341,11 @@ export class BinarySession ): Promise | undefined> { const live = await this.#retainedClientLive(retained.client, options); if (isAborted(options.signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), + ); if (!live) return undefined; if (options.snapshot !== undefined) { const imported = this.importAnalysisSnapshot(options.snapshot); @@ -158,6 +380,7 @@ export class BinarySession /** External live state cannot be replayed by a direct or composed CLI workflow. */ allowsSnapshotReplay(operation: AnalysisOperation): boolean { + if (this.#activeBundleResources !== undefined) return false; const capabilities = this.#active?.route.capabilities; const descriptor = capabilities?.get(operation); if (descriptor !== undefined) return descriptor.cachePolicy !== "live"; @@ -201,7 +424,13 @@ export class BinarySession options: BinarySessionOpenOptions = {}, ): Promise> { if (isAborted(options.signal)) - return Promise.resolve(err(new AnalysisCancelledError("open_binary"))); + return Promise.resolve( + err( + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), + ), + ); return resolveSessionTarget({ router: this.#providerRouter, current: this.#active, @@ -285,7 +514,11 @@ export class BinarySession ): Promise> { if (options.loadSnapshot === undefined) return ok(options); if (isAborted(options.signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), + ); const loaded = await options.loadSnapshot(); if (!loaded.ok) return loaded; const { loadSnapshot: _loaded, ...rest } = options; @@ -300,13 +533,21 @@ export class BinarySession if (this.#externalMutationCleanupFailure?.cleanupIncomplete === true) return err(this.#externalMutationCleanupFailure); if (isAborted(options.signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), + ); const resolved = await resolve(); if (!resolved.ok) return resolved; const { target, route, sameTarget } = resolved.value; const { profile } = route; if (isAborted(options.signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), + ); const activeProfile = this.#active?.profile; const sameProfile = activeProfile === null || activeProfile === undefined @@ -314,11 +555,22 @@ export class BinarySession : profile !== null && analysisProfilesEqual(activeProfile, profile); await this.#drainCalls(admittedThrough); if (isAborted(options.signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), + ); const pendingClosed = await this.#retryPendingCleanup(); if (!pendingClosed.ok) return pendingClosed; + const pendingResourcesClosed = + await this.#retryPendingBundleResourceCleanup(); + if (!pendingResourcesClosed.ok) return pendingResourcesClosed; if (isAborted(options.signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), + ); const retained = sameTarget && sameProfile ? this.#active : undefined; if (retained !== undefined) { const fastPath = await this.#tryRetainedFastPath( @@ -363,10 +615,43 @@ export class BinarySession await this.#failedOpen( run, previous, - new AnalysisCancelledError("open_binary"), + new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }), ), ); } + const cleanupFailures = trackCleanupFailures(options.signal, "open_binary"); + if (this.#activeBundleResources !== undefined) { + this.#pendingBundleResourceCleanup.push(this.#activeBundleResources); + this.#activeBundleResources = undefined; + const retiredResources = await this.#retryPendingBundleResourceCleanup( + undefined, + cleanupFailures.onFailure, + ); + if (!retiredResources.ok) { + const primary = cleanupFailures.error ?? retiredResources.error; + const failed = await this.#failedOpen( + run, + previous, + primary, + cleanupFailures.onFailure, + ); + return err(cleanupFailures.error ?? failed); + } + } + if (isAborted(options.signal)) { + const cancelled = new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + }); + const failed = await this.#failedOpen( + run, + previous, + cancelled, + cleanupFailures.onFailure, + ); + return err(cleanupFailures.error ?? failed); + } const profilelessWorkflowProfile = profile === null ? profilelessNativeDispatchProfile(route) : undefined; this.#active = { @@ -409,6 +694,14 @@ export class BinarySession ) => Promise>, ): Promise, AnalysisError>> { return this.#serialize(async (admittedThrough) => { + if (this.#activeBundleResources !== undefined) + return err( + new AnalysisCapabilityUnavailableError( + "bundle-resources", + operationName, + "External mutation requires an active executable analysis target.", + ), + ); if (isAborted(signal)) return err(new AnalysisCancelledError(operationName)); await this.#drainCalls(admittedThrough); @@ -499,14 +792,20 @@ export class BinarySession readonly retainProviderDocuments?: boolean; }, clearRecords = true, + onCleanupFailure?: (failure: AnalysisError) => void, ): Promise> { const previous = this.#active; + const previousResources = this.#activeBundleResources; this.#active = undefined; + this.#activeBundleResources = undefined; + if (previousResources !== undefined) + this.#pendingBundleResourceCleanup.push(previousResources); const closed = previous === undefined ? await this.#retryPendingCleanup( options.progress, options.retainProviderDocuments, + onCleanupFailure, ) : await this.#retireClient(previous, { ...(options.progress === undefined @@ -515,12 +814,20 @@ export class BinarySession ...(options.retainProviderDocuments === true ? { retainDocument: true } : {}), + ...(onCleanupFailure === undefined ? {} : { onCleanupFailure }), }); if (clearRecords) this.clearSessionRecords(); this.#clearRuntimeAvailability(); const fileResources = await this.closeWriteResources(); + if (!fileResources.ok) onCleanupFailure?.(fileResources.error); const candidateResources = await this.#providerRouter.closeOwnedResources(); + if (!candidateResources.ok) onCleanupFailure?.(candidateResources.error); + const bundleClosed = await this.#retryPendingBundleResourceCleanup( + options.progress, + onCleanupFailure, + ); const resourceFailures = [ + ...(bundleClosed.ok ? [] : [bundleClosed.error]), ...(fileResources.ok ? [] : [fileResources.error]), ...(candidateResources.ok ? [] : [candidateResources.error]), ]; @@ -544,22 +851,61 @@ export class BinarySession return resourceFailure === undefined ? ok(null) : err(resourceFailure); } + async #retryPendingBundleResourceCleanup( + progress?: ExecutionOptions["progress"], + onCleanupFailure?: (failure: AnalysisError) => void, + ): Promise> { + if (this.#pendingBundleResourceCleanup.length === 0) return ok(null); + const pending = this.#pendingBundleResourceCleanup; + const retained: ActiveBundleResourceSession[] = []; + let failure: AnalysisError | undefined; + for (const session of pending) { + const closed = await closeAnalysisClient( + session.client, + "bundle-resources", + { + ...(progress === undefined ? {} : { progress }), + ...(onCleanupFailure === undefined ? {} : { onCleanupFailure }), + }, + ); + if (!closed.ok) { + retained.push(session); + failure = + failure === undefined + ? closed.error + : analysisErrorWithCleanupFailure( + failure, + closed.error, + "close_binary", + ); + } + } + this.#pendingBundleResourceCleanup = retained; + return failure === undefined ? ok(null) : err(failure); + } + async #retireClient( run: SessionRun, options: Pick & { readonly retainDocument?: boolean; + readonly onCleanupFailure?: (failure: AnalysisError) => void; } = {}, ): Promise> { this.#pendingCleanup = { run, retainDocument: options.retainDocument === true, }; - return this.#retryPendingCleanup(options.progress); + return this.#retryPendingCleanup( + options.progress, + undefined, + options.onCleanupFailure, + ); } async #retryPendingCleanup( progress?: ExecutionOptions["progress"], retainDocument?: boolean, + onCleanupFailure?: (failure: AnalysisError) => void, ): Promise> { let pending = this.#pendingCleanup; if (pending === undefined) return ok(null); @@ -573,6 +919,7 @@ export class BinarySession { ...(progress === undefined ? {} : { progress }), ...(pending.retainDocument ? { retainDocument: true } : {}), + ...(onCleanupFailure === undefined ? {} : { onCleanupFailure }), }, ); if (closed.ok) this.#pendingCleanup = undefined; @@ -583,8 +930,12 @@ export class BinarySession run: SessionRun, previous: SessionBinding | undefined, primary: AnalysisError, + onCleanupFailure?: (failure: AnalysisError) => void, ): Promise { - const closed = await this.#retireClient(run); + const closed = await this.#retireClient( + run, + onCleanupFailure === undefined ? {} : { onCleanupFailure }, + ); if (!closed.ok) return analysisErrorWithCleanupFailure( primary, @@ -606,11 +957,22 @@ export class BinarySession /** Describe the active binary session. */ status(options: { readonly signal?: AbortSignal } = {}): Promise { - const run = this.#active ?? this.#pendingCleanup?.run; + const resourceRun = this.#activeBundleResources; + const run = + this.#active ?? + resourceRun ?? + this.#pendingBundleResourceCleanup[0] ?? + this.#pendingCleanup?.run; const client = run?.client; return binarySessionStatus({ ...options, target: this.#active?.target, + ...(resourceRun === undefined + ? {} + : { + bundleResourceRoot: resourceRun.root, + resourceCapabilities: resourceRun.capabilities, + }), route: this.#currentRoute(), router: this.#providerRouter, runtimeUnavailability: this.#runtimeUnavailability, @@ -620,6 +982,7 @@ export class BinarySession providerOperationHealth: client?.operationHealthSnapshot?.() ?? null, cleanupPending: this.#pendingCleanup !== undefined || + this.#pendingBundleResourceCleanup.length > 0 || this.hasPendingFileResources || this.#providerRouter.hasPendingOwnedResources(), }); @@ -632,6 +995,31 @@ export class BinarySession : structuredClone(this.#active.target); } + activeBundleResourceRoot(): ResolvedAppBundleResourceRoot | undefined { + const root = this.#activeBundleResources?.root; + return root === undefined ? undefined : structuredClone(root); + } + + override exportAnalysisSnapshot() { + if (this.#activeBundleResources !== undefined) + return err( + new AnalysisSnapshotMismatchError( + "Analysis snapshots do not apply to a resource-only app session because it has no executable identity or replay binding.", + ), + ); + return super.exportAnalysisSnapshot(); + } + + override importAnalysisSnapshot(snapshot: AnalysisSnapshot) { + if (this.#activeBundleResources !== undefined) + return err( + new AnalysisSnapshotMismatchError( + "Analysis snapshots do not apply to a resource-only app session because it has no executable identity or replay binding.", + ), + ); + return super.importAnalysisSnapshot(snapshot); + } + protected activeAnalysisBinding() { return this.#active; } @@ -723,6 +1111,21 @@ export class BinarySession transition, ); if (!transitioned.ok) return transitioned; + const resources = this.#activeBundleResources; + if (resources !== undefined) { + if (!BUNDLE_RESOURCE_OPERATIONS.has(name)) + return err( + new AnalysisCapabilityUnavailableError( + "bundle-resources", + name, + "This operation requires an executable analysis target; the active session contains only app resource access.", + ), + ); + const result = await resources.client.execute(name, arguments_, options); + if (!result.ok) return result; + const execution = { ...result.value, subject: null }; + return ok(execution); + } const prepared = prepareSessionExecution({ active: this.#active, operation: name, @@ -783,7 +1186,9 @@ export class BinarySession name !== "trace_dylib_resolution" ) { const evidence = createEvidence( - execution.value.subject ?? active.target, + execution.value.subject === undefined + ? active.target + : (execution.value.subject ?? undefined), execution.value.provider, { operation: name, diff --git a/src/application/binary/BinarySessionBundleResources.test.ts b/src/application/binary/BinarySessionBundleResources.test.ts new file mode 100644 index 0000000000..376fa51ee0 --- /dev/null +++ b/src/application/binary/BinarySessionBundleResources.test.ts @@ -0,0 +1,799 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; + +import { + createAnalysisExecution, + type AnalysisClient, + type ProviderIdentity, +} from "../AnalysisProvider.js"; +import { parseBinaryTarget } from "../BinaryTargetResolver.js"; +import { InvestigationRecords } from "../investigation/InvestigationRecords.js"; +import { AnalysisProviderRegistry } from "./AnalysisProviderRegistry.js"; +import { BinarySession } from "./BinarySession.js"; +import { SessionProviderRouter } from "./SessionProviderRouter.js"; +import { bundleResourceRootSchema } from "../../domain/apple/bundleResourceRoot.js"; +import { AnalysisSnapshotMismatchError } from "../../domain/evidenceErrors.js"; +import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { err } from "../../domain/result.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { createEvidence } from "../../domain/evidence.js"; +import { createAnalysisProfile } from "../../domain/analysisProfile.js"; +import { ok } from "../../domain/result.js"; +import { createTestProviderRouter } from "../../../tests/fixtures/binarySession.js"; + +const identity: ProviderIdentity = { + id: "bundle-resource-fixture", + name: "Bundle resource fixture", + version: "1", +}; +const binaryIdentity: ProviderIdentity = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", +}; +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +it("opens a resource-only bundle without executable identity and keeps snapshot operations inapplicable", async () => { + const rootPath = await mkdtemp(join(tmpdir(), "rea-resource-session-")); + temporaryDirectories.push(rootPath); + await writeFile( + join(rootPath, "Info.plist"), + '', + ); + let openedRoot: string | undefined; + let closed = false; + const client: AnalysisClient = { + execute: async (operation) => { + if (operation === "health") + return { ok: true, value: createAnalysisExecution(null, identity) }; + return { + ok: true, + value: createAnalysisExecution( + { operation, target_sha256: null }, + identity, + ), + }; + }, + close: async () => { + closed = true; + return { ok: true, value: null }; + }, + }; + const session = new BinarySession( + SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []), + new InvestigationRecords(), + { + createClient: (root) => { + openedRoot = root.canonicalPath; + return { client, capabilities: [] }; + }, + }, + ); + + const opened = await session.openBundleResources(rootPath); + expect(opened.ok).toBe(true); + if (!opened.ok) throw opened.error; + expect(opened.value.executableSha256).toBeNull(); + expect(openedRoot).toBe(opened.value.canonicalPath); + expect( + bundleResourceRootSchema.parse({ + caller_path: opened.value.callerPath, + canonical_path: opened.value.canonicalPath, + info_plist_path: opened.value.infoPlistPath, + layout: opened.value.layout, + executable_sha256: opened.value.executableSha256, + }).executable_sha256, + ).toBeNull(); + + const result = await session.execute("inspect_plist", {}); + expect(result.ok).toBe(true); + if (result.ok) expect(result.value.subject).toBeNull(); + const binaryOnly = await session.execute("binary_overview", {}); + expect(binaryOnly.ok).toBe(false); + expect(await session.status()).toMatchObject({ + open: true, + session_kind: "bundle_resources", + resource_root: { canonical_path: opened.value.canonicalPath }, + }); + const snapshot = session.exportAnalysisSnapshot(); + expect(snapshot.ok).toBe(false); + if (!snapshot.ok) + expect(snapshot.error).toBeInstanceOf(AnalysisSnapshotMismatchError); + const imported = session.importAnalysisSnapshot({} as AnalysisSnapshot); + expect(imported.ok).toBe(false); + if (!imported.ok) + expect(imported.error).toBeInstanceOf(AnalysisSnapshotMismatchError); + const closedResult = await session.close(); + expect(closedResult.ok).toBe(true); + expect(closed).toBe(true); +}); + +it("keeps the exact caller reason for cancellation before resource-root resolution", async () => { + const rootPath = await mkdtemp(join(tmpdir(), "rea-resource-pre-cancel-")); + temporaryDirectories.push(rootPath); + const reason = { origin: "caller-pre-aborted" }; + const controller = new AbortController(); + controller.abort(reason); + let clientCreated = false; + const session = new BinarySession( + SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []), + new InvestigationRecords(), + { + createClient: () => { + clientCreated = true; + throw new Error("pre-aborted call must not create a client"); + }, + }, + ); + const result = await session.openBundleResources(rootPath, { + signal: controller.signal, + }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError", cause: reason }, + }); + expect(clientCreated).toBe(false); +}); + +it("retains ownership when an unhealthy resource client cannot be closed", async () => { + const rootPath = await mkdtemp(join(tmpdir(), "rea-resource-cleanup-")); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + let closeAttempts = 0; + const client: AnalysisClient = { + execute: async () => err(new ProviderAdapterError(identity.id, "health")), + close: async () => { + closeAttempts += 1; + return closeAttempts === 1 + ? err(new ProviderAdapterError(identity.id, "close_binary")) + : { ok: true, value: null }; + }, + }; + const session = new BinarySession( + SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []), + new InvestigationRecords(), + { createClient: () => ({ client, capabilities: [] }) }, + ); + + const opened = await session.openBundleResources(rootPath); + expect(opened.ok).toBe(false); + expect(closeAttempts).toBe(1); + expect(await session.status()).toMatchObject({ cleanup_pending: true }); + expect(await session.status()).toMatchObject({ open: false }); + + const closed = await session.close(); + expect(closed.ok).toBe(true); + expect(closeAttempts).toBe(2); + expect(await session.status()).toMatchObject({ cleanup_pending: false }); +}); + +it("cancels a resource switch during prior-client retirement and closes its candidate", async () => { + const rootPath = await mkdtemp(join(tmpdir(), "rea-resource-cancel-switch-")); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + + let closeStarted!: () => void; + const closing = new Promise((resolve) => { + closeStarted = resolve; + }); + let releaseClose!: () => void; + const closeGate = new Promise((resolve) => { + releaseClose = resolve; + }); + let created = 0; + let candidateCloseCalls = 0; + const session = new BinarySession( + SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []), + new InvestigationRecords(), + { + createClient: () => { + created += 1; + const candidate = created > 1; + return { + client: { + execute: async () => ({ + ok: true, + value: createAnalysisExecution(null, identity), + }), + close: async () => { + if (candidate) candidateCloseCalls += 1; + else { + closeStarted(); + await closeGate; + } + return { ok: true, value: null }; + }, + }, + capabilities: [], + }; + }, + }, + ); + + expect((await session.openBundleResources(rootPath)).ok).toBe(true); + const controller = new AbortController(); + const reason = { phase: "prior-resource-retirement" }; + const opening = session.openBundleResources(rootPath, { + signal: controller.signal, + }); + await Promise.race([ + closing, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("prior resource client close did not start")), + 1_000, + ), + ), + ]); + controller.abort(reason); + releaseClose(); + + const result = await opening; + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + if (!result.ok) expect(result.error).toBeInstanceOf(AnalysisCancelledError); + if (!result.ok) expect(result.error.cause).toBe(reason); + expect(candidateCloseCalls).toBe(1); + expect(await session.status()).toMatchObject({ open: false }); + expect(await session.close()).toMatchObject({ ok: true }); +}); + +it("keeps cancellation primary when resource retirement fails and retries the owner", async () => { + const rootPath = await mkdtemp( + join(tmpdir(), "rea-resource-cancel-close-failure-"), + ); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + let closeStarted!: () => void; + const closing = new Promise((resolve) => { + closeStarted = resolve; + }); + let releaseClose!: () => void; + const closeGate = new Promise((resolve) => { + releaseClose = resolve; + }); + let created = 0; + let oldCloseAttempts = 0; + let candidateCloseAttempts = 0; + const session = new BinarySession( + SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []), + new InvestigationRecords(), + { + createClient: () => { + created += 1; + const candidate = created > 1; + return { + client: { + execute: async () => ({ + ok: true, + value: createAnalysisExecution(null, identity), + }), + close: async () => { + if (candidate) { + candidateCloseAttempts += 1; + return { ok: true, value: null }; + } + oldCloseAttempts += 1; + if (oldCloseAttempts === 1) { + closeStarted(); + await closeGate; + } + return oldCloseAttempts < 3 + ? err(new ProviderAdapterError(identity.id, "close_binary")) + : { ok: true, value: null }; + }, + }, + capabilities: [], + }; + }, + }, + ); + + expect((await session.openBundleResources(rootPath)).ok).toBe(true); + const controller = new AbortController(); + const reason = { phase: "resource-close-failed" }; + const opening = session.openBundleResources(rootPath, { + signal: controller.signal, + }); + await Promise.race([ + closing, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("resource close did not start")), + 1_000, + ), + ), + ]); + controller.abort(reason); + releaseClose(); + + const result = await opening; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failed resource switch"); + expect(result.error.cleanupPrimary).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cleanupPrimary?.cause).toBe(reason); + expect(result.error.cleanupIncomplete).toBe(true); + expect(candidateCloseAttempts).toBe(1); + expect(oldCloseAttempts).toBe(2); + expect(await session.status()).toMatchObject({ + open: false, + cleanup_pending: true, + }); + expect(await session.close()).toMatchObject({ ok: true }); + expect(oldCloseAttempts).toBe(3); +}); + +it("keeps an earlier resource close failure primary when a later cleanup is cancelled", async () => { + const rootPath = await mkdtemp( + join(tmpdir(), "rea-resource-close-before-cancel-"), + ); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + let candidateCloseStarted!: () => void; + const candidateClosing = new Promise((resolve) => { + candidateCloseStarted = resolve; + }); + let releaseCandidateClose!: () => void; + const candidateCloseGate = new Promise((resolve) => { + releaseCandidateClose = resolve; + }); + let created = 0; + let oldCloseAttempts = 0; + const originalFailure = new ProviderAdapterError(identity.id, "close_binary"); + const session = new BinarySession( + SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []), + new InvestigationRecords(), + { + createClient: () => { + created += 1; + const candidate = created > 1; + return { + client: { + execute: async () => ({ + ok: true, + value: createAnalysisExecution(null, identity), + }), + close: async () => { + if (candidate) { + candidateCloseStarted(); + await candidateCloseGate; + return { ok: true, value: null }; + } + oldCloseAttempts += 1; + return oldCloseAttempts === 1 + ? err(originalFailure) + : { ok: true, value: null }; + }, + }, + capabilities: [], + }; + }, + }, + ); + + expect((await session.openBundleResources(rootPath)).ok).toBe(true); + const controller = new AbortController(); + const reason = { phase: "after-physical-close-failure" }; + const opening = session.openBundleResources(rootPath, { + signal: controller.signal, + }); + await Promise.race([ + candidateClosing, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("candidate close did not start")), + 1_000, + ), + ), + ]); + controller.abort(reason); + releaseCandidateClose(); + + const result = await opening; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failed resource switch"); + expect(result.error).toBe(originalFailure); + expect(result.error.cleanupPrimary).toBeUndefined(); + expect(oldCloseAttempts).toBe(2); + expect(await session.close()).toMatchObject({ ok: true }); +}); + +it("keeps cancellation primary when binary retirement fails before opening resources", async () => { + const rootPath = await mkdtemp( + join(tmpdir(), "rea-binary-close-cancel-resource-"), + ); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + let closeStarted!: () => void; + const closing = new Promise((resolve) => { + closeStarted = resolve; + }); + let releaseClose!: () => void; + const closeGate = new Promise((resolve) => { + releaseClose = resolve; + }); + let binaryCloseAttempts = 0; + let resourceCloseAttempts = 0; + const binaryClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" ? null : "fixture-name", + binaryIdentity, + ), + }), + close: async () => { + binaryCloseAttempts += 1; + if (binaryCloseAttempts === 1) { + closeStarted(); + await closeGate; + return err(new ProviderAdapterError(binaryIdentity.id, "close_binary")); + } + return { ok: true, value: null }; + }, + }; + const resourceClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" ? null : { operation, target_sha256: null }, + identity, + { subject: null }, + ), + }), + close: async () => { + resourceCloseAttempts += 1; + return { ok: true, value: null }; + }, + }; + const session = new BinarySession( + createTestProviderRouter(() => binaryClient, { + resolveAnalysisProfile: async () => + ok({ + profile: createAnalysisProfile( + { ...binaryIdentity, version: binaryIdentity.version ?? "1" }, + { fixture: true }, + ), + }), + }), + new InvestigationRecords(), + { + createClient: () => ({ client: resourceClient, capabilities: [] }), + }, + ); + expect((await session.open(process.execPath)).ok).toBe(true); + expect((await session.execute("address_name", {})).ok).toBe(true); + + const controller = new AbortController(); + const reason = { phase: "binary-retirement" }; + const opening = session.openBundleResources(rootPath, { + signal: controller.signal, + }); + await Promise.race([ + closing, + new Promise((_, reject) => + setTimeout(() => reject(new Error("binary close did not start")), 1_000), + ), + ]); + controller.abort(reason); + releaseClose(); + + const result = await opening; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failed resource open"); + expect(result.error.cleanupPrimary).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cleanupPrimary?.cause).toBe(reason); + expect(result.error.cleanupIncomplete).toBe(true); + expect(resourceCloseAttempts).toBe(1); + expect(await session.status()).toMatchObject({ + open: false, + cleanup_pending: true, + }); + expect(await session.close()).toMatchObject({ ok: true }); + expect(binaryCloseAttempts).toBe(2); +}); + +it("keeps an earlier resource close failure primary when binary candidate cleanup is cancelled", async () => { + const rootPath = await mkdtemp( + join(tmpdir(), "rea-resource-close-before-binary-cancel-"), + ); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + let candidateCloseStarted!: () => void; + const candidateClosing = new Promise((resolve) => { + candidateCloseStarted = resolve; + }); + let releaseCandidateClose!: () => void; + const candidateCloseGate = new Promise((resolve) => { + releaseCandidateClose = resolve; + }); + let resourceCloseAttempts = 0; + let binaryCloseAttempts = 0; + const originalFailure = new ProviderAdapterError(identity.id, "close_binary"); + const resourceClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" ? null : { operation, target_sha256: null }, + identity, + { subject: null }, + ), + }), + close: async () => { + resourceCloseAttempts += 1; + return resourceCloseAttempts === 1 + ? err(originalFailure) + : { ok: true, value: null }; + }, + }; + const binaryClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" ? null : "fixture-name", + binaryIdentity, + ), + }), + close: async () => { + binaryCloseAttempts += 1; + if (binaryCloseAttempts === 1) { + candidateCloseStarted(); + await candidateCloseGate; + } + return { ok: true, value: null }; + }, + }; + const router = createTestProviderRouter(() => binaryClient); + const route = router.initialRoute(); + const parsedTarget = await parseBinaryTarget(process.execPath); + if (!parsedTarget.ok) throw parsedTarget.error; + const session = new BinarySession(router, new InvestigationRecords(), { + createClient: () => ({ client: resourceClient, capabilities: [] }), + }); + expect((await session.openBundleResources(rootPath)).ok).toBe(true); + + const controller = new AbortController(); + const reason = { phase: "after-resource-close-failure" }; + const opening = session.openResolvedTarget( + { target: parsedTarget.value, route }, + { signal: controller.signal }, + ); + await Promise.race([ + candidateClosing, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("binary candidate close did not start")), + 1_000, + ), + ), + ]); + controller.abort(reason); + releaseCandidateClose(); + + const result = await opening; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failed binary open"); + expect(result.error).toBe(originalFailure); + expect(result.error.cleanupPrimary).toBeUndefined(); + expect(binaryCloseAttempts).toBe(1); + expect(resourceCloseAttempts).toBe(1); + expect(await session.close()).toMatchObject({ ok: true }); + expect(resourceCloseAttempts).toBe(2); +}); + +it("keeps an earlier binary close failure primary when candidate cleanup is cancelled", async () => { + const rootPath = await mkdtemp( + join(tmpdir(), "rea-binary-close-before-cancel-"), + ); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + let candidateCloseStarted!: () => void; + const candidateClosing = new Promise((resolve) => { + candidateCloseStarted = resolve; + }); + let releaseCandidateClose!: () => void; + const candidateCloseGate = new Promise((resolve) => { + releaseCandidateClose = resolve; + }); + let binaryCloseAttempts = 0; + let resourceCloseAttempts = 0; + const originalFailure = new ProviderAdapterError( + binaryIdentity.id, + "close_binary", + ); + const binaryClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" ? null : "fixture-name", + binaryIdentity, + ), + }), + close: async () => { + binaryCloseAttempts += 1; + return binaryCloseAttempts === 1 + ? err(originalFailure) + : { ok: true, value: null }; + }, + }; + const resourceClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" ? null : { operation, target_sha256: null }, + identity, + { subject: null }, + ), + }), + close: async () => { + resourceCloseAttempts += 1; + if (resourceCloseAttempts === 1) { + candidateCloseStarted(); + await candidateCloseGate; + } + return { ok: true, value: null }; + }, + }; + const session = new BinarySession( + createTestProviderRouter(() => binaryClient, { + resolveAnalysisProfile: async () => + ok({ + profile: createAnalysisProfile( + { ...binaryIdentity, version: binaryIdentity.version ?? "1" }, + { fixture: true }, + ), + }), + }), + new InvestigationRecords(), + { + createClient: () => ({ client: resourceClient, capabilities: [] }), + }, + ); + expect((await session.open(process.execPath)).ok).toBe(true); + expect((await session.execute("address_name", {})).ok).toBe(true); + + const controller = new AbortController(); + const reason = { phase: "after-binary-close-failure" }; + const opening = session.openBundleResources(rootPath, { + signal: controller.signal, + }); + await Promise.race([ + candidateClosing, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("resource candidate close did not start")), + 1_000, + ), + ), + ]); + controller.abort(reason); + releaseCandidateClose(); + + const result = await opening; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failed resource open"); + expect(result.error).toBe(originalFailure); + expect(result.error.cleanupPrimary).toBeUndefined(); + expect(binaryCloseAttempts).toBe(1); + expect(resourceCloseAttempts).toBe(1); + expect(await session.close()).toMatchObject({ ok: true }); + expect(binaryCloseAttempts).toBe(2); +}); + +it("cancels a binary switch during resource retirement and retains null-subject Evidence", async () => { + const rootPath = await mkdtemp(join(tmpdir(), "rea-resource-cancel-binary-")); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + let resourceCloseStarted!: () => void; + const closing = new Promise((resolve) => { + resourceCloseStarted = resolve; + }); + let releaseResourceClose!: () => void; + const closeGate = new Promise((resolve) => { + releaseResourceClose = resolve; + }); + let resourceCloseAttempts = 0; + const binaryClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" + ? null + : { document: "fixture", procedure_count: 0, string_count: 0 }, + binaryIdentity, + ), + }), + close: async () => ({ ok: true, value: null }), + }; + const resourceClient: AnalysisClient = { + execute: async (operation) => ({ + ok: true, + value: createAnalysisExecution( + operation === "health" ? null : { operation, target_sha256: null }, + identity, + { subject: null }, + ), + }), + close: async () => { + resourceCloseAttempts += 1; + resourceCloseStarted(); + if (resourceCloseAttempts === 1) { + await closeGate; + return err(new ProviderAdapterError(identity.id, "close_binary")); + } + return { ok: true, value: null }; + }, + }; + const session = new BinarySession( + createTestProviderRouter(() => binaryClient, { + resolveAnalysisProfile: async () => + ok({ + profile: createAnalysisProfile( + { ...binaryIdentity, version: binaryIdentity.version ?? "1" }, + { fixture: true }, + ), + }), + }), + new InvestigationRecords(), + { + createClient: () => ({ client: resourceClient, capabilities: [] }), + }, + ); + expect((await session.openBundleResources(rootPath)).ok).toBe(true); + const observed = await session.execute("inspect_plist", {}); + expect(observed.ok).toBe(true); + if (!observed.ok) throw observed.error; + const recorded = session.recordEvidence( + createEvidence(undefined, identity, { + operation: "inspect_plist", + parameters: {}, + result: observed.value.result, + }), + ); + expect(recorded.ok).toBe(true); + const recordsBefore = session.exportEvidenceBundle().records; + expect(recordsBefore).toHaveLength(1); + const evidenceId = recordsBefore[0]?.evidence_id; + if (evidenceId === undefined) throw new Error("Expected retained Evidence"); + + const controller = new AbortController(); + const reason = { phase: "resource-retirement" }; + const opening = session.open(process.execPath, { signal: controller.signal }); + await Promise.race([ + closing, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("resource client close did not start")), + 1_000, + ), + ), + ]); + controller.abort(reason); + releaseResourceClose(); + + const result = await opening; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancelled binary open"); + expect(result.error.cleanupPrimary).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cleanupPrimary?.cause).toBe(reason); + expect(result.error.cleanupIncomplete).toBe(true); + expect(await session.status()).toMatchObject({ open: false }); + expect(session.exportEvidenceBundle().records).toHaveLength(1); + expect(session.evidenceById(evidenceId)).toMatchObject({ subject: null }); + expect(await session.status()).toMatchObject({ cleanup_pending: true }); + expect(await session.close()).toMatchObject({ ok: true }); + expect(resourceCloseAttempts).toBe(2); +}); diff --git a/src/application/binary/BinarySessionComposition.ts b/src/application/binary/BinarySessionComposition.ts index 6131a84cd3..233754696a 100644 --- a/src/application/binary/BinarySessionComposition.ts +++ b/src/application/binary/BinarySessionComposition.ts @@ -3,6 +3,7 @@ import type { AnalysisProvider } from "../AnalysisProvider.js"; import { AnalysisProviderRegistry } from "./AnalysisProviderRegistry.js"; import { BinarySession } from "./BinarySession.js"; import { SessionProviderRouter } from "./SessionProviderRouter.js"; +import type { BundleResourceClientFactory } from "./BundleResourceSession.js"; /** * Compose one provider-neutral session from deep-provider selection and @@ -11,8 +12,10 @@ import { SessionProviderRouter } from "./SessionProviderRouter.js"; export const composeBinarySession = ( registry: AnalysisProviderRegistry, auxiliaryProviders: readonly AnalysisProvider[] = [], + bundleResourceFactory?: BundleResourceClientFactory, ): BinarySession => new BinarySession( SessionProviderRouter.selectable(registry, auxiliaryProviders), new InvestigationRecords(), + bundleResourceFactory, ); diff --git a/src/application/binary/BinarySessionExecution.ts b/src/application/binary/BinarySessionExecution.ts index 73dfe4eeae..7f42ba3259 100644 --- a/src/application/binary/BinarySessionExecution.ts +++ b/src/application/binary/BinarySessionExecution.ts @@ -134,6 +134,7 @@ export const bindExecutionTarget = ( : { architecture: target.architecture }), }; if ( + result.value.subject !== undefined && result.value.subject !== null && result.value.subject.sha256 !== target.sha256 && !isArtifactBundleSubject(operation, result.value.subject, target) @@ -144,7 +145,9 @@ export const bindExecutionTarget = ( `${operation}:subject`, ), ); - return ok({ ...result.value, subject: result.value.subject ?? subject }); + return result.value.subject === undefined + ? ok({ ...result.value, subject }) + : result; }; const isArtifactBundleSubject = ( diff --git a/src/application/binary/BinarySessionLiveness.test.ts b/src/application/binary/BinarySessionLiveness.test.ts index 59b056a970..c7455328be 100644 --- a/src/application/binary/BinarySessionLiveness.test.ts +++ b/src/application/binary/BinarySessionLiveness.test.ts @@ -40,6 +40,39 @@ import { SessionProviderRouter } from "./SessionProviderRouter.js"; * `operationHealthSnapshot`, the way `HopperClient.operationHealth` reports a * recorded process exit. */ +it("binds omitted provider subjects to the selected target", async () => { + const { session } = openSession(); + try { + const opened = await session.open(process.execPath); + expect(opened.ok).toBe(true); + const execution = await session.execute("address_name", {}); + expect(execution.ok).toBe(true); + if (!execution.ok || !opened.ok) return; + expect(execution.value.subject).toEqual({ + path: opened.value.path, + sha256: opened.value.sha256, + format: opened.value.format, + ...(opened.value.architecture === undefined + ? {} + : { architecture: opened.value.architecture }), + }); + } finally { + await session.close(); + } +}); + +it("preserves an explicitly unbound provider observation", async () => { + const { session } = openSession({ explicitNullSubject: true }); + try { + expect((await session.open(process.execPath)).ok).toBe(true); + const execution = await session.execute("address_name", {}); + expect(execution.ok).toBe(true); + if (execution.ok) expect(execution.value.subject).toBeNull(); + } finally { + await session.close(); + } +}); + describe("BinarySession open liveness", () => { it("retires a locally blocked owner even when its native health is idle", async () => { const { session, deep } = openSession(); @@ -349,6 +382,7 @@ const openSession = ( operationHealth?: boolean; failOperationInitially?: boolean; beforeFirstClose?: () => Promise; + explicitNullSubject?: boolean; } = {}, ): { session: BinarySession; @@ -383,6 +417,7 @@ const killableDeepProvider = (options: { operationHealth?: boolean; failOperationInitially?: boolean; beforeFirstClose?: () => Promise; + explicitNullSubject?: boolean; }): { provider: AnalysisProviderCandidate; created: KillableChild[]; @@ -433,6 +468,10 @@ const killableDeepProvider = (options: { createAnalysisExecution( operation === "health" ? null : `liveness:${operation}`, deepIdentity, + operation !== "health" && + options.explicitNullSubject === true + ? { subject: null } + : {}, ), ), ) diff --git a/src/application/binary/BinarySessionPort.ts b/src/application/binary/BinarySessionPort.ts index 7e8dee1098..dfabfee771 100644 --- a/src/application/binary/BinarySessionPort.ts +++ b/src/application/binary/BinarySessionPort.ts @@ -13,6 +13,7 @@ import type { Result } from "../../domain/result.js"; import type { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import type { AnalysisSnapshotLoader } from "./BinarySessionOpen.js"; import type { WorkflowSnapshotRecordInput } from "./BinarySessionRecords.js"; +import type { ResolvedAppBundleResourceRoot } from "../AppBundleExecutable.js"; import type { AnalysisExecution, @@ -76,6 +77,11 @@ export interface BinarySessionPort readonly providerId?: AnalysisProviderSelector; }, ): Promise>; + /** Open a resource-only app root without selecting or inventing an executable target. */ + openBundleResources( + path: string, + options?: Pick, + ): Promise>; close( options?: Pick & { readonly retainProviderDocuments?: boolean; diff --git a/src/application/binary/BinarySessionStatus.ts b/src/application/binary/BinarySessionStatus.ts index f18358e6d6..104d7e1011 100644 --- a/src/application/binary/BinarySessionStatus.ts +++ b/src/application/binary/BinarySessionStatus.ts @@ -1,4 +1,5 @@ import type { BinaryTarget } from "../../domain/binaryTargetTypes.js"; +import type { ResolvedAppBundleResourceRoot } from "../AppBundleExecutable.js"; import { binaryTargetSummary } from "./BinaryTargetSummary.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import type { ProviderOperationHealth } from "../../domain/providerOperationHealth.js"; @@ -20,6 +21,8 @@ import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js" interface BinarySessionStatusInput { readonly target: BinaryTarget | undefined; + readonly bundleResourceRoot?: ResolvedAppBundleResourceRoot; + readonly resourceCapabilities?: readonly CapabilityDescriptor[]; readonly route: SessionProviderRoute; readonly router: Pick; readonly runtimeUnavailability: ReadonlyMap< @@ -37,6 +40,8 @@ interface BinarySessionStatusInput { /** Project internal provider routing state into the caller-visible session status. */ export const binarySessionStatus = async ({ target, + bundleResourceRoot, + resourceCapabilities = [], route, router, runtimeUnavailability, @@ -47,7 +52,11 @@ export const binarySessionStatus = async ({ cleanupPending = false, signal, }: BinarySessionStatusInput): Promise => { - const routeCapabilities = [...route.capabilities.values()] + const routeCapabilities = ( + bundleResourceRoot === undefined + ? [...route.capabilities.values()] + : [...resourceCapabilities] + ) .filter(({ operation }) => operation !== "inventory_artifact") .sort((left, right) => compareUnicodeCodePoints(left.operation, right.operation), @@ -64,29 +73,32 @@ export const binarySessionStatus = async ({ }); const capabilities = providerCapabilitySets(routeCapabilities); const analysisProviderBinding = - route.binding === null + bundleResourceRoot !== undefined ? null - : { - provider: providerSummary(route.binding.identity), - selection_source: route.binding.selectionSource, - analysis_profile: structuredClone(route.binding.profile), - }; - const analysisProviderCandidates = ( - await router.candidateStatuses(route, signal) - ).map((candidate) => ({ - provider: providerSummary(candidate.provider), - availability: structuredClone(candidate.availability), - target_support: structuredClone(candidate.targetSupport), - selected: candidate.selected, - capabilities: providerCapabilitySets( - candidate.capabilities - .slice() - .filter(({ operation }) => operation !== "inventory_artifact") - .sort((left, right) => - compareUnicodeCodePoints(left.operation, right.operation), - ), - ), - })); + : route.binding === null + ? null + : { + provider: providerSummary(route.binding.identity), + selection_source: route.binding.selectionSource, + analysis_profile: structuredClone(route.binding.profile), + }; + const analysisProviderCandidates = + bundleResourceRoot !== undefined + ? [] + : (await router.candidateStatuses(route, signal)).map((candidate) => ({ + provider: providerSummary(candidate.provider), + availability: structuredClone(candidate.availability), + target_support: structuredClone(candidate.targetSupport), + selected: candidate.selected, + capabilities: providerCapabilitySets( + candidate.capabilities + .slice() + .filter(({ operation }) => operation !== "inventory_artifact") + .sort((left, right) => + compareUnicodeCodePoints(left.operation, right.operation), + ), + ), + })); const common = { cleanup_pending: cleanupPending, capabilities, @@ -105,7 +117,20 @@ export const binarySessionStatus = async ({ ), }; return target === undefined - ? { open: false, ...common } + ? bundleResourceRoot === undefined + ? { open: false, ...common } + : { + open: true, + ...common, + session_kind: "bundle_resources", + resource_root: { + caller_path: bundleResourceRoot.callerPath, + canonical_path: bundleResourceRoot.canonicalPath, + info_plist_path: bundleResourceRoot.infoPlistPath, + layout: bundleResourceRoot.layout, + executable_sha256: bundleResourceRoot.executableSha256, + }, + } : { open: true, ...common, diff --git a/src/application/binary/BundleResourceSession.test.ts b/src/application/binary/BundleResourceSession.test.ts new file mode 100644 index 0000000000..4f51ee5096 --- /dev/null +++ b/src/application/binary/BundleResourceSession.test.ts @@ -0,0 +1,106 @@ +import { expect, it } from "vitest"; + +import { combineBundleResourceClients } from "./BundleResourceSession.js"; +import { + createAnalysisExecution, + type AnalysisClient, + type ProviderIdentity, +} from "../AnalysisProvider.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { err } from "../../domain/result.js"; + +const identity: ProviderIdentity = { + id: "bundle-resource-close-fixture", + name: "Bundle resource close fixture", + version: "1", +}; + +it("retains both child close failures when a resource owner rejects", async () => { + const attempts: string[] = []; + const artifact: AnalysisClient = { + execute: async () => ({ + ok: true, + value: createAnalysisExecution(null, identity), + }), + close: async () => { + attempts.push("artifact"); + throw new Error("artifact close rejected"); + }, + }; + const native: AnalysisClient = { + execute: async () => ({ + ok: true, + value: createAnalysisExecution(null, identity), + }), + close: async () => { + attempts.push("native"); + throw new Error("native close rejected"); + }, + }; + + const closed = await combineBundleResourceClients(artifact, native).close(); + expect(attempts.sort()).toEqual(["artifact", "native"]); + expect(closed.ok).toBe(false); + if (!closed.ok) expect(closed.error.cleanupIncomplete).toBe(true); +}); + +it("reports the first mixed child failure before awaiting the other owner and retries both", async () => { + let artifactCloseAttempts = 0; + let nativeCloseAttempts = 0; + let nativeCloseStarted!: () => void; + const nativeStarted = new Promise((resolve) => { + nativeCloseStarted = resolve; + }); + let releaseNativeClose!: () => void; + const nativeCloseGate = new Promise((resolve) => { + releaseNativeClose = resolve; + }); + let cleanupFailureObserved!: (failure: Error) => void; + const failureObserved = new Promise((resolve) => { + cleanupFailureObserved = resolve; + }); + const artifact: AnalysisClient = { + execute: async () => ({ + ok: true, + value: createAnalysisExecution(null, identity), + }), + close: async () => { + artifactCloseAttempts += 1; + return artifactCloseAttempts === 1 + ? err( + new ProviderCleanupError(identity.id, ["artifact-reader"], { + reason: "artifact owner remains open", + }), + ) + : { ok: true, value: null }; + }, + }; + const native: AnalysisClient = { + execute: async () => ({ + ok: true, + value: createAnalysisExecution(null, identity), + }), + close: async () => { + nativeCloseAttempts += 1; + if (nativeCloseAttempts === 1) { + nativeCloseStarted(); + await nativeCloseGate; + } + return { ok: true, value: null }; + }, + }; + const combined = combineBundleResourceClients(artifact, native); + const closing = combined.close({ + onCleanupFailure: cleanupFailureObserved, + }); + await Promise.all([nativeStarted, failureObserved]); + releaseNativeClose(); + const first = await closing; + expect(first.ok).toBe(false); + if (first.ok) throw new Error("Expected mixed close failure"); + expect(first.error.cleanupIncomplete).toBe(true); + + expect(await combined.close()).toMatchObject({ ok: true }); + expect(artifactCloseAttempts).toBe(2); + expect(nativeCloseAttempts).toBe(2); +}); diff --git a/src/application/binary/BundleResourceSession.ts b/src/application/binary/BundleResourceSession.ts new file mode 100644 index 0000000000..a511238312 --- /dev/null +++ b/src/application/binary/BundleResourceSession.ts @@ -0,0 +1,85 @@ +import type { + AnalysisClient, + AnalysisOperation, + CapabilityDescriptor, +} from "../AnalysisProvider.js"; +import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { err, ok } from "../../domain/result.js"; +import type { ResolvedAppBundleResourceRoot } from "../AppBundleExecutable.js"; +import { closeAnalysisClient } from "../../providers/AnalysisClientCleanup.js"; + +export const BUNDLE_NATIVE_RESOURCE_OPERATIONS = new Set([ + "inspect_plist", +]); + +export const BUNDLE_ARTIFACT_RESOURCE_OPERATIONS = new Set([ + "decode_interface_builder", + "inspect_asset_catalog", + "inspect_keyed_archive", +]); + +export const BUNDLE_RESOURCE_OPERATIONS = new Set([ + ...BUNDLE_NATIVE_RESOURCE_OPERATIONS, + ...BUNDLE_ARTIFACT_RESOURCE_OPERATIONS, +]); + +/** Provider-specific owner of the four read-only app-resource operations. */ +export interface BundleResourceClientFactory { + createClient( + root: ResolvedAppBundleResourceRoot, + ): BundleResourceClientBinding | Promise; +} + +export interface BundleResourceClientBinding { + readonly client: AnalysisClient; + readonly capabilities: readonly CapabilityDescriptor[]; +} + +export interface ActiveBundleResourceSession { + readonly root: ResolvedAppBundleResourceRoot; + readonly client: AnalysisClient; + readonly capabilities: readonly CapabilityDescriptor[]; + readonly runId: string; +} + +/** Route each resource read to the owner that understands its format. */ +export const combineBundleResourceClients = ( + artifactClient: AnalysisClient, + nativeClient: AnalysisClient, +): AnalysisClient => ({ + execute: (operation, parameters, options) => { + if (operation === "health") + return artifactClient.execute(operation, parameters, options); + if (!BUNDLE_RESOURCE_OPERATIONS.has(operation)) + return Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + "bundle-resources", + operation, + "This session provides read-only app resource inspection only.", + ), + ), + ); + return BUNDLE_NATIVE_RESOURCE_OPERATIONS.has(operation) + ? nativeClient.execute(operation, parameters, options) + : artifactClient.execute(operation, parameters, options); + }, + close: async (options) => { + const [artifact, native] = await Promise.all([ + closeAnalysisClient(artifactClient, "bundle-resources", options), + closeAnalysisClient(nativeClient, "bundle-resources", options), + ]); + if (!artifact.ok && !native.ok) + return err( + analysisErrorWithCleanupFailure( + artifact.error, + native.error, + "close_binary", + ), + ); + if (!artifact.ok) return artifact; + if (!native.ok) return native; + return ok(null); + }, +}); diff --git a/src/application/firmware/FirmwareAnalysisService.ts b/src/application/firmware/FirmwareAnalysisService.ts index 765a9da13d..0d3344646a 100644 --- a/src/application/firmware/FirmwareAnalysisService.ts +++ b/src/application/firmware/FirmwareAnalysisService.ts @@ -46,7 +46,7 @@ export class FirmwareAnalysisService { const executed = await this.provider.execute(parsed.data, options); if (!executed.ok) return executed; const execution = executed.value; - if (execution.subject === null) + if (execution.subject == null) return err( new AnalysisOutputError( operation, diff --git a/src/application/javascript/JavaScriptRecoveryService.ts b/src/application/javascript/JavaScriptRecoveryService.ts index 687b71d67d..4a5b282159 100644 --- a/src/application/javascript/JavaScriptRecoveryService.ts +++ b/src/application/javascript/JavaScriptRecoveryService.ts @@ -82,7 +82,7 @@ export class JavaScriptRecoveryService { const result = javascriptRecoveryResultSchema.safeParse( execution.value.result, ); - if (!result.success || execution.value.subject === null) + if (!result.success || execution.value.subject == null) return err( new AnalysisOutputError( OPERATION, diff --git a/src/application/native/NativeValueTrace.ts b/src/application/native/NativeValueTrace.ts index f7ff79200c..92564a18fc 100644 --- a/src/application/native/NativeValueTrace.ts +++ b/src/application/native/NativeValueTrace.ts @@ -113,7 +113,7 @@ export const traceNativeValues = async ( continue; } const parsed = functionDossierSchema.safeParse(execution.value.result); - if (!parsed.success || execution.value.subject === null) + if (!parsed.success || execution.value.subject == null) return err( new AnalysisOutputError( "trace_native_values", diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts index 7f8aaeb73e..eddc109ac8 100644 --- a/src/application/sqlite/SqliteDatabaseService.ts +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -95,7 +95,7 @@ export class SqliteDatabaseService { const report = sqliteDatabaseSchema.safeParse(execution.result); if ( !report.success || - execution.subject === null || + execution.subject == null || !matchesSelectedDatabase(report.data, input.data, execution.subject) ) return err( diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index 446a824334..2a115286bc 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -44,6 +44,9 @@ import { createEvidence } from "../domain/evidence.js"; import { freezeJsonSnapshot } from "../domain/immutableJson.js"; import { createArtifactInspection } from "../domain/artifactInspection.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import type { ResolvedAppBundleResourceRoot } from "../application/AppBundleExecutable.js"; +import type { BundleResourceRoot } from "../domain/apple/bundleResourceRoot.js"; +import { BUNDLE_ARTIFACT_RESOURCE_OPERATIONS } from "../application/binary/BundleResourceSession.js"; /** Read-only inventory and exclusively owned extraction provider. */ export class ArtifactProvider implements AnalysisProvider { @@ -71,6 +74,120 @@ export class ArtifactProvider implements AnalysisProvider { createClient(target: BinaryTarget): AnalysisClient { return new ArtifactClient(target, this.environment, this.#platform); } + + createBundleResourceClient( + root: ResolvedAppBundleResourceRoot, + ): AnalysisClient { + return new BundleResourceArtifactClient( + root, + this.environment, + this.#platform, + ); + } +} + +const serializedResourceRoot = ( + root: ResolvedAppBundleResourceRoot, +): BundleResourceRoot => ({ + caller_path: root.callerPath, + canonical_path: root.canonicalPath, + info_plist_path: root.infoPlistPath, + layout: root.layout, + executable_sha256: root.executableSha256, +}); + +class BundleResourceArtifactClient implements AnalysisClient { + readonly #resourceScope = new ArtifactResourceScope(); + + constructor( + private readonly root: ResolvedAppBundleResourceRoot, + private readonly environment: Readonly, + private readonly platform: NodeJS.Platform, + ) {} + + async execute( + operation: AnalysisOperation, + parameters: Readonly>, + options?: ExecutionOptions, + ) { + if (operation === "health") + return ok(createAnalysisExecution(null, ARTIFACT_GRAPH_PROVIDER)); + if (!BUNDLE_ARTIFACT_RESOURCE_OPERATIONS.has(operation)) + return err( + new AnalysisCapabilityUnavailableError( + ARTIFACT_GRAPH_PROVIDER.id, + operation, + "The app resource reader supports only plist, Interface Builder, asset catalog, and keyed archive inspection.", + ), + ); + try { + const resourceRoot = serializedResourceRoot(this.root); + const result = + operation === "decode_interface_builder" + ? await analyzeInterfaceBuilderBundle({ + bundlePath: this.root.canonicalPath, + targetSha256: null, + resourceRoot, + limits: interfaceBuilderLimitsSchema.parse(parameters), + ...(options?.signal === undefined + ? {} + : { signal: options.signal }), + }) + : operation === "inspect_keyed_archive" + ? await inspectBundleKeyedArchive({ + bundlePath: this.root.canonicalPath, + targetSha256: null, + resourceRoot, + parameters, + platform: this.platform, + ...(options?.signal === undefined + ? {} + : { signal: options.signal }), + }) + : operation === "inspect_asset_catalog" + ? await analyzeAppleAssetCatalogs({ + bundlePath: this.root.canonicalPath, + targetSha256: null, + resourceRoot, + environment: this.environment, + page: parameters, + ...(options?.signal === undefined + ? {} + : { signal: options.signal }), + }) + : (() => { + throw new AnalysisCapabilityUnavailableError( + ARTIFACT_GRAPH_PROVIDER.id, + operation, + "Property list inspection requires a platform-native reader.", + ); + })(); + return ok( + createAnalysisExecution(result, ARTIFACT_GRAPH_PROVIDER, { + limitations: "limitations" in result ? result.limitations : [], + }), + ); + } catch (cause: unknown) { + return err( + translateArtifactFailure(operation as ArtifactAnalysisOperation, cause), + ); + } + } + + close(): Promise> { + return this.#resourceScope.close().then( + () => ok(null), + (cause: unknown) => + err( + new ProviderCleanupError( + ARTIFACT_GRAPH_PROVIDER.id, + ["app bundle resource reader"], + { reason: cause instanceof Error ? cause.message : String(cause) }, + { cause, operation: "close_binary" }, + ), + ), + ); + } } class ArtifactClient implements AnalysisClient { @@ -225,7 +342,9 @@ class ArtifactClient implements AnalysisClient { }), ); } catch (cause: unknown) { - return err(translateArtifactFailure(operation, cause)); + return err( + translateArtifactFailure(operation as ArtifactAnalysisOperation, cause), + ); } } diff --git a/src/artifacts/BundleResourceArtifactClient.test.ts b/src/artifacts/BundleResourceArtifactClient.test.ts new file mode 100644 index 0000000000..90fd7ef12a --- /dev/null +++ b/src/artifacts/BundleResourceArtifactClient.test.ts @@ -0,0 +1,60 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { build } from "plist"; +import { afterEach, expect, it } from "vitest"; + +import { ArtifactProvider } from "./ArtifactProvider.js"; +import type { ResolvedAppBundleResourceRoot } from "../application/AppBundleExecutable.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +it("returns keyed archive evidence with root provenance and unknown executable identity", async () => { + const bundlePath = await mkdtemp(join(tmpdir(), "rea-resource-reader-")); + temporaryDirectories.push(bundlePath); + await mkdir(join(bundlePath, "Resources")); + await writeFile( + join(bundlePath, "Info.plist"), + '', + ); + const archive = build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "sample"], + $top: { root: { UID: 1 } }, + }); + await writeFile(join(bundlePath, "Resources", "Archive.plist"), archive); + const root: ResolvedAppBundleResourceRoot = { + callerPath: bundlePath, + canonicalPath: bundlePath, + infoPlistPath: join(bundlePath, "Info.plist"), + layout: "root", + executableSha256: null, + }; + const provider = new ArtifactProvider({}, "linux"); + const client = provider.createBundleResourceClient(root); + const execution = await client.execute("inspect_keyed_archive", { + path: "Resources/Archive.plist", + }); + expect(execution.ok).toBe(true); + if (execution.ok) { + expect(execution.value.result).toMatchObject({ + target_sha256: null, + resource_root: { + caller_path: bundlePath, + canonical_path: bundlePath, + executable_sha256: null, + }, + archive_path: "Resources/Archive.plist", + }); + } + expect((await client.close()).ok).toBe(true); +}); diff --git a/src/artifacts/apple/AppleAssetCatalogAnalysis.ts b/src/artifacts/apple/AppleAssetCatalogAnalysis.ts index 84809ec08b..0388a69c17 100644 --- a/src/artifacts/apple/AppleAssetCatalogAnalysis.ts +++ b/src/artifacts/apple/AppleAssetCatalogAnalysis.ts @@ -21,7 +21,8 @@ const MAX_TOTAL_METADATA_BYTES = 64 * 1024 * 1024; /** Inspect compiled asset catalog metadata without extracting rendition bytes. */ export const analyzeAppleAssetCatalogs = async (input: { readonly bundlePath: string; - readonly targetSha256: string; + readonly targetSha256: string | null; + readonly resourceRoot?: import("../../domain/apple/bundleResourceRoot.js").BundleResourceRoot; readonly environment: Readonly; readonly page?: unknown; readonly signal?: AbortSignal; @@ -46,6 +47,9 @@ export const analyzeAppleAssetCatalogs = async (input: { )({ bundlePath: input.bundlePath, targetSha256: input.targetSha256, + ...(input.resourceRoot === undefined + ? {} + : { resourceRoot: input.resourceRoot }), signal: input.signal === undefined ? interfaceBuilderCancel.signal @@ -71,6 +75,9 @@ export const analyzeAppleAssetCatalogs = async (input: { ); const result = projectAppleAssetCatalogPage({ targetSha256: input.targetSha256, + ...(input.resourceRoot === undefined + ? {} + : { resourceRoot: input.resourceRoot }), catalogs, offset: page.offset, limit: page.limit, diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index f05c03b780..26efffab26 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -55,7 +55,8 @@ const MAX_REPORTED_OMITTED_ARCHIVES = 64; /** Decode compiled Interface Builder archives from a local app bundle. */ export const analyzeInterfaceBuilderBundle = async (input: { readonly bundlePath: string; - readonly targetSha256: string; + readonly targetSha256: string | null; + readonly resourceRoot?: import("../../domain/apple/bundleResourceRoot.js").BundleResourceRoot; readonly limits?: unknown; readonly signal?: AbortSignal; }) => { @@ -205,6 +206,9 @@ export const analyzeInterfaceBuilderBundle = async (input: { } const result = buildInterfaceBuilderAnalysis({ targetSha256: input.targetSha256, + ...(input.resourceRoot === undefined + ? {} + : { resourceRoot: input.resourceRoot }), toolVersion: "rea-interface-builder/1", documents, limits, diff --git a/src/artifacts/apple/KeyedArchiveReader.ts b/src/artifacts/apple/KeyedArchiveReader.ts index 5081bdf01d..a15b8cfbe5 100644 --- a/src/artifacts/apple/KeyedArchiveReader.ts +++ b/src/artifacts/apple/KeyedArchiveReader.ts @@ -152,7 +152,8 @@ const keyedArchiveKindRemediation = ( /** Read exactly one regular, contained bundle entry without following symlinks. */ export const inspectBundleKeyedArchive = async (input: { bundlePath: string; - targetSha256: string; + targetSha256: string | null; + resourceRoot?: import("../../domain/apple/bundleResourceRoot.js").BundleResourceRoot; parameters: unknown; signal?: AbortSignal; /** Host whose available workflows the remediation names. */ @@ -228,6 +229,9 @@ export const inspectBundleKeyedArchive = async (input: { try { return keyedArchiveResultSchema.parse({ target_sha256: input.targetSha256, + ...(input.resourceRoot === undefined + ? {} + : { resource_root: input.resourceRoot }), archive_path: entry.path, archive_sha256: createHash("sha256").update(bytes).digest("hex"), ...decodeKeyedArchiveBytes(bytes, selected), diff --git a/src/cliBundleResource.test.ts b/src/cliBundleResource.test.ts new file mode 100644 index 0000000000..f366a0cb2a --- /dev/null +++ b/src/cliBundleResource.test.ts @@ -0,0 +1,69 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { build } from "plist"; +import { afterEach, expect, it } from "vitest"; + +import { createCli } from "./cli.js"; +import { parseEvidence } from "./domain/evidence.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +it("inspects a keyless app bundle through the public CLI and preserves resource provenance", async () => { + const parent = await mkdtemp(join(tmpdir(), "rea-bundle-resource-cli-")); + temporaryDirectories.push(parent); + const bundle = join(parent, "Fixture.app"); + const resources = join(bundle, "Resources"); + await mkdir(resources, { recursive: true }); + await writeFile(join(bundle, "Info.plist"), ""); + await writeFile( + join(resources, "Archive.plist"), + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "resource-only-value"], + $top: { root: { UID: 1 } }, + }), + ); + + const cli = createCli({ REA_LOG_LEVEL: "silent" }); + let stdout = ""; + let exitCode = 0; + await cli.serve( + ["inspect-keyed-archive", bundle, "Resources/Archive.plist", "--json"], + { + env: {}, + exit: (code) => { + exitCode = code; + }, + stdout: (text) => { + stdout += text; + }, + }, + ); + expect(exitCode, stdout).toBe(0); + expect(parseEvidence(JSON.parse(stdout))).toMatchObject({ + subject: null, + normalized_result: { + archive_path: "Resources/Archive.plist", + target_sha256: null, + resource_root: { + caller_path: bundle, + canonical_path: bundle, + executable_sha256: null, + }, + objects: expect.arrayContaining([ + expect.objectContaining({ value: "resource-only-value" }), + ]), + }, + }); +}); diff --git a/src/composition/binary.ts b/src/composition/binary.ts index 03fb2aacd7..43f99bb46c 100644 --- a/src/composition/binary.ts +++ b/src/composition/binary.ts @@ -12,6 +12,11 @@ import { AnalysisProviderRegistry } from "../application/binary/AnalysisProvider import { composeBinarySession } from "../application/binary/BinarySessionComposition.js"; import { LazyAnalysisProvider } from "../application/binary/LazyAnalysisProvider.js"; import { ManagedStaticProvider } from "../dotnet/ManagedStaticProvider.js"; +import { + BUNDLE_ARTIFACT_RESOURCE_OPERATIONS, + BUNDLE_NATIVE_RESOURCE_OPERATIONS, + combineBundleResourceClients, +} from "../application/binary/BundleResourceSession.js"; /** * Compose the target-switching runtime shared directly by CLI and MCP adapters. @@ -67,9 +72,49 @@ const composeDeepBinarySession = ( auxiliaryDeclarations.map( (declaration) => new LazyAnalysisProvider(declaration), ), + createBundleResourceFactory(environment, process.platform), ); }; +const createBundleResourceFactory = ( + environment: Readonly, + platform: NodeJS.Platform, +) => { + return { + createClient: async ( + root: import("../application/AppBundleExecutable.js").ResolvedAppBundleResourceRoot, + ) => { + const [{ ArtifactProvider }, { NativeMacOSProvider }] = await Promise.all( + [ + import("../artifacts/ArtifactProvider.js"), + import("../native/NativeMacOSProvider.js"), + ], + ); + const artifact = new ArtifactProvider(environment, platform); + const native = new NativeMacOSProvider(environment, undefined, platform); + const capabilities = [ + ...artifact + .capabilities() + .filter(({ operation }) => + BUNDLE_ARTIFACT_RESOURCE_OPERATIONS.has(operation), + ), + ...native + .capabilities() + .filter(({ operation }) => + BUNDLE_NATIVE_RESOURCE_OPERATIONS.has(operation), + ), + ]; + return { + client: combineBundleResourceClients( + artifact.createBundleResourceClient(root), + native.createBundleResourceClient(root), + ), + capabilities, + }; + }, + }; +}; + /** Compose an execution-free managed session without native provider selection. */ export const createManagedBinarySession = (): BinarySession => composeBinarySession(new AnalysisProviderRegistry([]), [ @@ -82,12 +127,19 @@ export const createArtifactBinarySession = ( platform: NodeJS.Platform = process.platform, ): Promise => import("../artifacts/ArtifactProvider.js").then(({ ArtifactProvider }) => - composeBinarySession(new AnalysisProviderRegistry([]), [ - new ArtifactProvider( + composeBinarySession( + new AnalysisProviderRegistry([]), + [ + new ArtifactProvider( + snapshotEnvironment(selectedEnvironment, platform), + platform, + ), + ], + createBundleResourceFactory( snapshotEnvironment(selectedEnvironment, platform), platform, ), - ]), + ), ); /** Compose native inspection utilities without candidates for deep analysis. */ @@ -96,11 +148,18 @@ export const createNativeBinarySession = ( platform: NodeJS.Platform = process.platform, ): Promise => import("../native/NativeMacOSProvider.js").then(({ NativeMacOSProvider }) => - composeBinarySession(new AnalysisProviderRegistry([]), [ - new NativeMacOSProvider( + composeBinarySession( + new AnalysisProviderRegistry([]), + [ + new NativeMacOSProvider( + snapshotEnvironment(selectedEnvironment, platform), + undefined, + platform, + ), + ], + createBundleResourceFactory( snapshotEnvironment(selectedEnvironment, platform), - undefined, platform, ), - ]), + ), ); diff --git a/src/contracts/sessionLifecycleInputs.ts b/src/contracts/sessionLifecycleInputs.ts index 4b03c1f66d..dba92baa75 100644 --- a/src/contracts/sessionLifecycleInputs.ts +++ b/src/contracts/sessionLifecycleInputs.ts @@ -65,6 +65,11 @@ export const openBinaryInputSchema = z }); }); +/** Open only app resources when no declared executable is available. */ +export const openBundleResourcesInputSchema = z.strictObject({ + path: binaryTargetPathSchema, +}); + /** Input contract for closing a target after an optional atomic snapshot. */ export const closeBinaryInputSchema = z.strictObject({ snapshot_path: snapshotPathSchema.optional(), diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index f43f4802de..4bb7e44e1a 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -17,6 +17,7 @@ import { sessionOutputSchemas } from "./toolOutputSchemaGroups.js"; import { requireOutputSchema } from "./toolOutputSchemaPrimitives.js"; import { closeBinaryInputSchema, + openBundleResourcesInputSchema, openBinaryInputSchema, } from "./sessionLifecycleInputs.js"; import { binarySessionInputSchema } from "./sessionStatusContract.js"; @@ -67,14 +68,19 @@ export const SESSION_TOOL_CONTRACTS = [ "Open a local executable, application bundle, archive, JavaScript, source map, plist, or analysis database after validation. format=dos-com explicitly interprets 1..65280 headerless bytes as a DOS COM analysis image. format=raw-binary interprets headerless bytes with a required architecture and entry_point; file offset 0 maps at that address. Omission preserves header-based detection. provider_id selects one deep provider or deterministic auto selection; the binding remains stable until close or an explicit switch, with no failure fallback. Reopening the same target and profile retains its active database and metadata edits. An optional analysis snapshot is imported atomically and must match the binary identity, concrete provider, and canonical analysis profile exactly; importing into a session with metadata mutations requires closing and reopening it first. Call close_binary when the investigation is finished.", openBinaryInputSchema, ), + session( + "open_bundle_resources", + "Open a local app bundle for read-only plist, Interface Builder, asset catalog, and keyed archive inspection without resolving or inventing its executable identity. This switches the shared session to a resource-only lane; executable analysis is unavailable until another target is opened.", + openBundleResourcesInputSchema, + ), session( "close_binary", - "When the investigation is finished, drain earlier provider requests, optionally write a provider-neutral analysis snapshot atomically to the caller-supplied path, then close the active target and every provider resource started for it. Later provider requests wait until this lifecycle operation finishes. Existing files require explicit overwrite; a failed save leaves the session open so cached analysis is not lost. Metadata edits prevent immutable snapshot saves until the session is recreated; export_evidence_bundle preserves mutable observations before closing without a snapshot.", + "When the investigation is finished, drain earlier provider requests, optionally write a provider-neutral analysis snapshot atomically to the caller-supplied path, then close the active analysis session and every provider resource started for it. Resource-only app sessions have no binary snapshot binding, so close them without snapshot_path. Later provider requests wait until this lifecycle operation finishes. Existing files require explicit overwrite; a failed save leaves the session open so cached analysis is not lost. Metadata edits prevent immutable snapshot saves until the session is recreated; export_evidence_bundle preserves mutable observations before closing without a snapshot.", closeBinaryInputSchema, ), session( "binary_session", - "Report the complete current target, provider, capability availability, client-feature, analysis, and server-identity status without starting analysis.", + "Report the active binary target or resource-only app root, provider and capability availability when applicable, client-feature, analysis, and server-identity status without starting analysis.", binarySessionInputSchema, ), session( diff --git a/src/contracts/toolContractExamples.ts b/src/contracts/toolContractExamples.ts index e2f96d1583..af218b43df 100644 --- a/src/contracts/toolContractExamples.ts +++ b/src/contracts/toolContractExamples.ts @@ -53,6 +53,7 @@ export const TOOL_EXAMPLE_OVERRIDES: Readonly< trace_call_path: { start: "0x1000", goal: "0x1100" }, trace_native_ui_action: { action: "buildTapped:" }, open_binary: { path: "/tmp/fixture" }, + open_bundle_resources: { path: "/tmp/Fixture.app" }, export_evidence_bundle: { path: "/tmp/evidence.json" }, get_evidence_bundle: { detail: "summary", diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 651a4245c1..0e11470c0a 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -679,6 +679,7 @@ export const TOOL_EFFECTS: Readonly> = { project_apple_application_graph: evidence, project_harmony_application_graph: evidence, open_binary: effects({ mutatesSession: true, launchesProcess: true }), + open_bundle_resources: effects({ mutatesSession: true }), close_binary: effects({ mutatesSession: true, writesFilesystem: true, diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index c1c1ba5413..cd1864c99c 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -26,6 +26,7 @@ import { nativeFunctionAnnotationsSchema } from "../domain/native/nativeFunction import { nativeLoadImageSchema } from "../domain/native/nativeLoadImage.js"; import { nativeUiResultSchema } from "../domain/native/nativeUiObservation.js"; import { nativeValueTraceSchema } from "../domain/native/nativeValueTrace.js"; +import { bundleResourceRootSchema } from "../domain/apple/bundleResourceRoot.js"; import { nativeDataTypeSchema } from "../domain/native/nativeDataType.js"; import { nativeCallObservationResultSchema } from "../domain/native/nativeCallObservation.js"; import { @@ -511,6 +512,7 @@ export const sessionOutputSchemas = { rawLifecycleTargetSchema, ]), ), + open_bundle_resources: lifecycleResultOf(bundleResourceRootSchema), close_binary: lifecycleResultOf( z.union([ z.null(), @@ -540,6 +542,11 @@ export const sessionOutputSchemas = { binary_session: lifecycleResultOf( z.union([ sessionProvider.extend({ open: z.literal(false) }), + sessionProvider.extend({ + open: z.literal(true), + session_kind: z.literal("bundle_resources"), + resource_root: bundleResourceRootSchema, + }), sessionProvider.extend({ open: z.literal(true), ...detectedLifecycleTargetSchema.shape, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index d991e789ed..c16b443ad6 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -78,6 +78,7 @@ import { jsonObjectSchema, jsonValueSchema } from "../jsonValue.js"; import { err, ok, type Result } from "../result.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { safeParseOwnedBinaryLayout } from "../native/binaryLayout.js"; +import { bundleResourceRootSchema } from "../apple/bundleResourceRoot.js"; import { functionDossierSchema } from "../hopperValues.js"; import { artifactGraphManifestSchema, @@ -346,7 +347,9 @@ const coverageSchema = z.strictObject({ const artifactSchema = z.strictObject({ path: z .string() - .describe("Exact recorded artifact path, including an empty string."), + .describe( + "Selected artifact path, or the canonical resource root when resource-only Evidence has no executable subject.", + ), sha256: digestSchema.nullable(), }); @@ -524,6 +527,7 @@ export const machoSummaryViewResultSchema = z.strictObject({ export const interfaceBuilderSummaryViewResultSchema = z.strictObject({ ...summaryViewResultBase, parent_operation: z.literal("decode_interface_builder"), + resource_root: bundleResourceRootSchema.optional(), summary: interfaceBuilderSummarySchema, }); @@ -782,6 +786,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ view: interfaceBuilderViewRequestSchema, items: interfaceBuilderViewItemsSchema, ...viewResultBase, + resource_root: bundleResourceRootSchema.optional(), ...interfaceBuilderViewContextSchema.shape, parent_operation: z.literal("decode_interface_builder"), }), diff --git a/src/domain/analysisView/interfaceBuilderView.test.ts b/src/domain/analysisView/interfaceBuilderView.test.ts index a591d3194a..b3f02916ea 100644 --- a/src/domain/analysisView/interfaceBuilderView.test.ts +++ b/src/domain/analysisView/interfaceBuilderView.test.ts @@ -1,11 +1,19 @@ import { describe, expect, it } from "vitest"; import { + analysisViewResultSchema, projectAnalysisView, type AnalysisViewParent, } from "./analysisView.js"; const target = "a".repeat(64); +const resourceRoot = { + caller_path: "/Applications/Keyless.app", + canonical_path: "/Applications/Keyless.app", + info_plist_path: "/Applications/Keyless.app/Info.plist", + layout: "root" as const, + executable_sha256: null, +}; const evidence = (artifact_path: string) => [ { kind: "interface_builder_resource" as const, @@ -68,6 +76,15 @@ const report = (truncated = false) => ({ }, limitations: ["Recognized archive fields only."], }); +const resourceReport = () => { + const decoded = report(); + return { + ...decoded, + target_sha256: null, + resource_root: resourceRoot, + graph: { ...decoded.graph, target_sha256: null }, + }; +}; const parent = ( normalizedResult: unknown = report(), sha256 = target, @@ -86,6 +103,33 @@ const parent = ( }); describe("Interface Builder analysis views", () => { + it("keeps resource-root provenance in both summary and detail views without a subject", () => { + const source: AnalysisViewParent = { + ...parent(resourceReport()), + subject: null, + }; + const summary = projectAnalysisView(source, { kind: "summary" }); + const detail = projectAnalysisView(source, { + kind: "interface-builder", + collection: "documents", + offset: 0, + limit: 64, + }); + if (!summary.ok) throw summary.error; + if (!detail.ok) throw detail.error; + + expect(analysisViewResultSchema.parse(summary.value)).toMatchObject({ + artifact: { path: resourceRoot.canonical_path, sha256: null }, + resource_root: resourceRoot, + summary: { target_sha256: null }, + }); + expect(analysisViewResultSchema.parse(detail.value)).toMatchObject({ + artifact: { path: resourceRoot.canonical_path, sha256: null }, + resource_root: resourceRoot, + items: report().documents, + }); + }); + it("summarizes documents and graph counts without rows", () => { const projected = projectAnalysisView(parent(), { kind: "summary" }); if (!projected.ok) throw projected.error; diff --git a/src/domain/analysisView/interfaceBuilderView.ts b/src/domain/analysisView/interfaceBuilderView.ts index 75a2c196ee..a4f49a92e4 100644 --- a/src/domain/analysisView/interfaceBuilderView.ts +++ b/src/domain/analysisView/interfaceBuilderView.ts @@ -6,10 +6,12 @@ import { interfaceBuilderAnalysisSchema, interfaceBuilderDocumentSchema, } from "../apple/interfaceBuilderGraph.js"; +import { bundleResourceRootSchema } from "../apple/bundleResourceRoot.js"; import { nativeInvestigationCoverageSchema, nativeInvestigationEdgeSchema, nativeInvestigationNodeSchema, + nativeInvestigationProviderSchema, } from "../native/nativeInvestigationGraph.js"; import { err, ok, type Result } from "../result.js"; import { parseAnalysisViewResult } from "./analysisViewParent.js"; @@ -67,9 +69,12 @@ const countsSchema = z.record(z.string(), z.number().int().positive()); /** Every decoded document with graph counts; rows and evidence stay retained. */ export const interfaceBuilderSummarySchema = z.strictObject({ - target_sha256: interfaceBuilderAnalysisSchema.shape.target_sha256, - provider: interfaceBuilderAnalysisSchema.shape.graph.shape.provider, - documents: interfaceBuilderAnalysisSchema.shape.documents, + target_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/u) + .nullable(), + provider: nativeInvestigationProviderSchema, + documents: interfaceBuilderDocumentSchema.array().max(64), available_counts: z.strictObject({ nodes: z.number().int().nonnegative(), edges: z.number().int().nonnegative(), @@ -132,7 +137,7 @@ const summarizeInterfaceBuilder = ( graph.edges.length + graph.coverage.length; return { - summary: { + summary: interfaceBuilderSummarySchema.parse({ target_sha256: report.target_sha256, provider: graph.provider, documents: report.documents, @@ -146,7 +151,7 @@ const summarizeInterfaceBuilder = ( coverage_statuses: tally(graph.coverage.map((row) => row.status)), truncated: graph.truncated, limitation_count: limitationCount, - }, + }), coverage: completeWithinViewCoverage(total, total), }; }; @@ -180,9 +185,15 @@ export const projectInterfaceBuilderView = ( parent_operation: "decode_interface_builder" as const, parent_digest: parent.evidenceId.slice(3), artifact: { - path: parent.subject?.local_path ?? "", + path: + parent.subject?.local_path ?? + report.resource_root?.canonical_path ?? + "", sha256: report.target_sha256, }, + ...(report.resource_root === undefined + ? {} + : { resource_root: report.resource_root }), limitations, unknowns: [ "Rows describe recognized archive objects and connections only; unrecognized classes, fields and runtime-created UI remain unknown.", diff --git a/src/domain/apple/appleAssetCatalog.test.ts b/src/domain/apple/appleAssetCatalog.test.ts index fdab85399c..ee8ee0d14c 100644 --- a/src/domain/apple/appleAssetCatalog.test.ts +++ b/src/domain/apple/appleAssetCatalog.test.ts @@ -1,11 +1,39 @@ import { describe, expect, it } from "vitest"; import { + appleAssetCatalogResultSchema, appleAssetCatalogInputSchema, parseAppleAssetCatalogRecords, projectAppleAssetCatalogPage, } from "./appleAssetCatalog.js"; +it("requires resource-root provenance when asset results have no executable identity", () => { + const base = { + target_sha256: null, + catalogs: [], + total_records: 0, + offset: 0, + limit: 1, + records: [], + resource_key_matches: [], + next_offset: null, + truncated: false, + limitations: [], + }; + const resource_root = { + caller_path: "/tmp/Fixture.app", + canonical_path: "/tmp/Fixture.app", + info_plist_path: "/tmp/Fixture.app/Info.plist", + layout: "root" as const, + executable_sha256: null, + }; + expect( + appleAssetCatalogResultSchema.parse({ ...base, resource_root }) + .resource_root, + ).toEqual(resource_root); + expect(appleAssetCatalogResultSchema.safeParse(base).success).toBe(false); +}); + describe("compiled Apple asset catalog projection", () => { it("preserves metadata and paginates across catalog boundaries", () => { const catalogs = [ diff --git a/src/domain/apple/appleAssetCatalog.ts b/src/domain/apple/appleAssetCatalog.ts index 1c89957976..81a96c3cc3 100644 --- a/src/domain/apple/appleAssetCatalog.ts +++ b/src/domain/apple/appleAssetCatalog.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { bundleResourceBoundResultSchema } from "./bundleResourceRoot.js"; import { jsonValueSchema } from "../jsonValue.js"; import { AnalysisOutputError } from "../analysisErrorCore.js"; @@ -66,18 +67,20 @@ const resourceKeyMatchSchema = z.strictObject({ }); /** Normalized result returned by asset catalog inspection. */ -export const appleAssetCatalogResultSchema = z.strictObject({ - target_sha256: z.string().regex(/^[a-f0-9]{64}$/u), - catalogs: z.array(catalogSchema), - total_records: z.number().int().nonnegative(), - offset: z.number().int().nonnegative(), - limit: z.number().int().positive(), - records: z.array(recordSchema), - resource_key_matches: z.array(resourceKeyMatchSchema), - next_offset: z.number().int().nonnegative().nullable(), - truncated: z.boolean(), - limitations: z.array(z.string()), -}); +export const appleAssetCatalogResultSchema = bundleResourceBoundResultSchema( + z.string().regex(/^[a-f0-9]{64}$/u), + { + catalogs: z.array(catalogSchema), + total_records: z.number().int().nonnegative(), + offset: z.number().int().nonnegative(), + limit: z.number().int().positive(), + records: z.array(recordSchema), + resource_key_matches: z.array(resourceKeyMatchSchema), + next_offset: z.number().int().nonnegative().nullable(), + truncated: z.boolean(), + limitations: z.array(z.string()), + }, +); export type AppleAssetCatalogRecord = z.infer; @@ -106,7 +109,8 @@ export const parseAppleAssetCatalogRecords = ( /** Project catalog records and apply stable global pagination. */ export const projectAppleAssetCatalogPage = (input: { - readonly targetSha256: string; + readonly targetSha256: string | null; + readonly resourceRoot?: import("./bundleResourceRoot.js").BundleResourceRoot; readonly catalogs: readonly { readonly path: string; readonly sha256: string; @@ -175,6 +179,9 @@ export const projectAppleAssetCatalogPage = (input: { }); return appleAssetCatalogResultSchema.parse({ target_sha256: input.targetSha256, + ...(input.resourceRoot === undefined + ? {} + : { resource_root: input.resourceRoot }), catalogs: input.catalogs.map( ({ path, sha256, records: catalogRecords }) => ({ path, diff --git a/src/domain/apple/bundleResourceRoot.ts b/src/domain/apple/bundleResourceRoot.ts new file mode 100644 index 0000000000..35cb5eeaee --- /dev/null +++ b/src/domain/apple/bundleResourceRoot.ts @@ -0,0 +1,33 @@ +import { z } from "zod"; + +/** Provenance for app resources, independent of executable file identity. */ +export const bundleResourceRootSchema = z.strictObject({ + caller_path: z.string().min(1), + canonical_path: z.string().min(1), + info_plist_path: z.string().min(1), + layout: z.enum(["contents", "root"]), + executable_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/u) + .nullable(), +}); + +export type BundleResourceRoot = z.infer; + +/** Output identity: executable-bound, or explicitly rooted resource-only. */ +export const bundleResourceBoundResultSchema = ( + executableSha256: z.ZodType, + fields: Fields, +) => + z.union([ + z.strictObject({ + target_sha256: executableSha256, + resource_root: bundleResourceRootSchema.optional(), + ...fields, + }), + z.strictObject({ + target_sha256: z.null(), + resource_root: bundleResourceRootSchema, + ...fields, + }), + ]); diff --git a/src/domain/apple/interfaceBuilderGraph.test.ts b/src/domain/apple/interfaceBuilderGraph.test.ts index 8f5bcc06b1..4b55b155bc 100644 --- a/src/domain/apple/interfaceBuilderGraph.test.ts +++ b/src/domain/apple/interfaceBuilderGraph.test.ts @@ -9,6 +9,33 @@ import { parseInterfaceBuilderRecords } from "./interfaceBuilderKeyedArchive.js" const hash = "a".repeat(64); +it("requires resource-root provenance when executable identity is unknown", () => { + const resourceRoot = { + caller_path: "/tmp/Fixture.app", + canonical_path: "/tmp/Fixture.app", + info_plist_path: "/tmp/Fixture.app/Info.plist", + layout: "root" as const, + executable_sha256: null, + }; + const result = buildInterfaceBuilderAnalysis({ + targetSha256: null, + resourceRoot, + toolVersion: "test", + documents: [], + limits: interfaceBuilderLimitsSchema.parse({}), + }); + expect(result.target_sha256).toBeNull(); + expect(result.resource_root).toEqual(resourceRoot); + expect(() => + buildInterfaceBuilderAnalysis({ + targetSha256: null, + toolVersion: "test", + documents: [], + limits: interfaceBuilderLimitsSchema.parse({}), + }), + ).toThrow(); +}); + describe("compiled Interface Builder graph projection", () => { it("normalizes object, hierarchy, action, outlet, and segue facts", () => { const raw = { diff --git a/src/domain/apple/interfaceBuilderGraph.ts b/src/domain/apple/interfaceBuilderGraph.ts index e38a6d326c..9f1409bd1b 100644 --- a/src/domain/apple/interfaceBuilderGraph.ts +++ b/src/domain/apple/interfaceBuilderGraph.ts @@ -1,4 +1,9 @@ import { z } from "zod"; +import { + bundleResourceRootSchema, + bundleResourceBoundResultSchema, + type BundleResourceRoot, +} from "./bundleResourceRoot.js"; import { createPlistNumberProjection, type PlistNumberLiterals, @@ -27,12 +32,14 @@ export const interfaceBuilderDocumentSchema = z.strictObject({ }); /** Normalized read-only Interface Builder graph with archive provenance. */ -export const interfaceBuilderAnalysisSchema = z.strictObject({ - target_sha256: z.string().regex(/^[a-f0-9]{64}$/u), - documents: z.array(interfaceBuilderDocumentSchema).max(64), - graph: nativeInvestigationGraphSchema, - limitations: z.array(z.string()), -}); +export const interfaceBuilderAnalysisSchema = bundleResourceBoundResultSchema( + z.string().regex(/^[a-f0-9]{64}$/u), + { + documents: z.array(interfaceBuilderDocumentSchema).max(64), + graph: nativeInvestigationGraphSchema, + limitations: z.array(z.string()), + }, +); export interface InterfaceBuilderDocumentInput { readonly relativePath: string; @@ -50,7 +57,8 @@ type InvestigationEdge = z.infer; /** Build the graph returned by the static Interface Builder archive decoder. */ export const buildInterfaceBuilderAnalysis = (input: { - readonly targetSha256: string; + readonly targetSha256: string | null; + readonly resourceRoot?: BundleResourceRoot; readonly toolVersion: string; readonly documents: readonly InterfaceBuilderDocumentInput[]; readonly limits: z.output; @@ -581,6 +589,9 @@ export const buildInterfaceBuilderAnalysis = (input: { }); return interfaceBuilderAnalysisSchema.parse({ target_sha256: input.targetSha256, + ...(input.resourceRoot === undefined + ? {} + : { resource_root: input.resourceRoot }), documents: summaries, graph: { target_sha256: input.targetSha256, diff --git a/src/domain/apple/keyedArchive.ts b/src/domain/apple/keyedArchive.ts index 32d0fbf29c..152de4f1a3 100644 --- a/src/domain/apple/keyedArchive.ts +++ b/src/domain/apple/keyedArchive.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { bundleResourceBoundResultSchema } from "./bundleResourceRoot.js"; import { AnalysisInputError } from "../analysisErrorCore.js"; import { jsonValueSchema, type JsonValue } from "../jsonValue.js"; import { localPathStringSchema } from "../localPath.js"; @@ -35,37 +36,44 @@ const linkSchema = z.strictObject({ raw: jsonValueSchema, }); /** Archive object identity is its original $objects index, never a recursive copy. */ -export const keyedArchiveResultSchema = z.strictObject({ - target_sha256: z.string(), - archive_path: z.string(), - archive_sha256: z.string(), - archive_format: z.enum(["binary-plist", "xml-plist"]), - roots: z.record(z.string(), jsonValueSchema), - objects: z.array( - z.strictObject({ - id: z.number().int().nonnegative(), - kind: z.enum([ - "nil", - "scalar", - "array", - "dictionary", - "class-descriptor", - "archived-object", - ]), - class_id: z.number().int().nonnegative().nullable(), - class_name: z.string().nullable(), - status: z.enum(["observed", "unknown-class", "unresolved-class", "nil"]), - value: jsonValueSchema, - }), - ), - references: z.array(linkSchema), - total_objects: z.number().int().nonnegative(), - total_references: z.number().int().nonnegative(), - offset: z.number().int().nonnegative(), - next_offset: z.number().int().nonnegative().nullable(), - truncated: z.boolean(), - limitations: z.array(z.string()), -}); +export const keyedArchiveResultSchema = bundleResourceBoundResultSchema( + z.string(), + { + archive_path: z.string(), + archive_sha256: z.string(), + archive_format: z.enum(["binary-plist", "xml-plist"]), + roots: z.record(z.string(), jsonValueSchema), + objects: z.array( + z.strictObject({ + id: z.number().int().nonnegative(), + kind: z.enum([ + "nil", + "scalar", + "array", + "dictionary", + "class-descriptor", + "archived-object", + ]), + class_id: z.number().int().nonnegative().nullable(), + class_name: z.string().nullable(), + status: z.enum([ + "observed", + "unknown-class", + "unresolved-class", + "nil", + ]), + value: jsonValueSchema, + }), + ), + references: z.array(linkSchema), + total_objects: z.number().int().nonnegative(), + total_references: z.number().int().nonnegative(), + offset: z.number().int().nonnegative(), + next_offset: z.number().int().nonnegative().nullable(), + truncated: z.boolean(), + limitations: z.array(z.string()), + }, +); const record = ( value: JsonValue | undefined, diff --git a/src/domain/native/nativeInspection.ts b/src/domain/native/nativeInspection.ts index f89f494c6e..434a75a1fa 100644 --- a/src/domain/native/nativeInspection.ts +++ b/src/domain/native/nativeInspection.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { bundleResourceRootSchema } from "../apple/bundleResourceRoot.js"; import { jsonValueSchema } from "../jsonValue.js"; const hexAddress = z.string().regex(/^0x[a-fA-F0-9]+$/u); @@ -170,6 +171,7 @@ export const inspectPlistSchema = z.object({ short_version: z.string().nullable(), }), source_path: z.string(), + resource_root: bundleResourceRootSchema.optional(), provenance: z.array(nativeCommandInvocationSchema), limitations: z.array(z.string()), }); diff --git a/src/domain/native/nativeInvestigationGraph.ts b/src/domain/native/nativeInvestigationGraph.ts index 9b9efc4128..89a00eeb5e 100644 --- a/src/domain/native/nativeInvestigationGraph.ts +++ b/src/domain/native/nativeInvestigationGraph.ts @@ -106,14 +106,19 @@ export const nativeInvestigationCoverageSchema = z.strictObject({ omitted: z.number().int().nonnegative(), }); +export const nativeInvestigationProviderSchema = z.strictObject({ + id: z.string().min(1), + version: z.string().nullable(), + tool_version: z.string().min(1), +}); + /** Complete provider-neutral graph bound to one immutable target identity. */ export const nativeInvestigationGraphSchema = z.strictObject({ - target_sha256: z.string().regex(/^[a-f0-9]{64}$/u), - provider: z.strictObject({ - id: z.string().min(1), - version: z.string().nullable(), - tool_version: z.string().min(1), - }), + target_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/u) + .nullable(), + provider: nativeInvestigationProviderSchema, nodes: z.array(nativeInvestigationNodeSchema), edges: z.array(nativeInvestigationEdgeSchema), coverage: z.array(nativeInvestigationCoverageSchema), @@ -137,8 +142,11 @@ export type NativeInvestigationTraceLimits = z.infer< /** Exact graph route and any unresolved edges encountered along the route. */ export const nativeInvestigationTraceSchema = z.strictObject({ - target_sha256: z.string().regex(/^[a-f0-9]{64}$/u), - provider: nativeInvestigationGraphSchema.shape.provider, + target_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/u) + .nullable(), + provider: nativeInvestigationProviderSchema, start: z.string().min(1), direction: z.enum(["forward", "backward"]), nodes: z.array(nativeInvestigationNodeSchema), diff --git a/src/domain/pcap/pcapEvidence.ts b/src/domain/pcap/pcapEvidence.ts index 848350fe36..d2379ac75f 100644 --- a/src/domain/pcap/pcapEvidence.ts +++ b/src/domain/pcap/pcapEvidence.ts @@ -4,7 +4,7 @@ import type { PcapRequest } from "./pcapAnalysis.js"; /** Domain facts needed by both successful and retained partial observations. */ interface PcapObservation { - readonly subject: Parameters[0] | null; + readonly subject?: Parameters[0] | null; readonly provider: Parameters[1]; readonly result: JsonValue; readonly rawResult: JsonValue | null; diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 7541e84398..f3e8da8e08 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -121,6 +121,8 @@ import { type NativeUiHelperRuntime, } from "./NativeUiHelperRuntime.js"; import { NativeUiInterruptionControl } from "./NativeUiInterruptionControl.js"; +import type { ResolvedAppBundleResourceRoot } from "../application/AppBundleExecutable.js"; +import { resolveBundleResourcePlistSelection } from "./NativePlistSource.js"; type SignatureRuntimeRootFactory = () => Promise< Pick @@ -166,6 +168,23 @@ export class NativeMacOSProvider implements AnalysisProvider { this.#privateRuntimeRootFactory, ); } + + createBundleResourceClient( + root: ResolvedAppBundleResourceRoot, + ): AnalysisClient { + const plistAvailable = this.#capabilities.some( + ({ operation, available }) => operation === "inspect_plist" && available, + ); + return new NativeMacOSClient( + undefined, + this.runner, + this.#tracerFactory(), + this.#environment, + this.#privateRuntimeRootFactory, + root, + plistAvailable, + ); + } } class NativeMacOSClient implements AnalysisClient { @@ -179,15 +198,23 @@ class NativeMacOSClient implements AnalysisClient { #closed = false; constructor( - private readonly target: BinaryTarget, + private readonly activeTarget: BinaryTarget | undefined, private readonly runner: NativeCommandRunner, private readonly tracer: NativeCallTracer, private readonly environment: Readonly, private readonly privateRuntimeRootFactory: | SignatureRuntimeRootFactory | undefined, + private readonly bundleResourceRoot?: ResolvedAppBundleResourceRoot, + private readonly bundlePlistAvailable = true, ) {} + private get target(): BinaryTarget { + if (this.activeTarget === undefined) + throw new Error("This provider client has no executable target"); + return this.activeTarget; + } + async execute( operation: AnalysisOperation, parameters: Readonly>, @@ -207,6 +234,22 @@ class NativeMacOSClient implements AnalysisClient { ); if (operation === "health") return ok(createAnalysisExecution(null, IDENTITY)); + if (this.activeTarget === undefined && operation !== "inspect_plist") + return err( + new AnalysisCapabilityUnavailableError( + IDENTITY.id, + operation, + "This client is bound to app resource access only.", + ), + ); + if (this.bundleResourceRoot !== undefined && !this.bundlePlistAvailable) + return err( + new AnalysisCapabilityUnavailableError( + IDENTITY.id, + "inspect_plist", + "Property list inspection requires a macOS host.", + ), + ); if (operation === "inspect_native_dispatch_metadata") { const input = toolContract(operation).inputSchema.safeParse(parameters); if (!input.success) @@ -806,10 +849,13 @@ class NativeMacOSClient implements AnalysisClient { try { return await this.#plistResources.run(async () => { try { - const plist = await resolveNativePlistSelection( - this.target, - requested, - ); + const plist = + this.bundleResourceRoot === undefined + ? await resolveNativePlistSelection(this.target, requested) + : resolveBundleResourcePlistSelection( + this.bundleResourceRoot, + requested, + ); if (!plist.ok) return plist; const fileInput = { selection: plist.value, @@ -898,6 +944,17 @@ class NativeMacOSClient implements AnalysisClient { value: parsed.value.value, bundle: parsed.value.bundle, source_path: plist.value.path, + ...(this.bundleResourceRoot === undefined + ? {} + : { + resource_root: { + caller_path: this.bundleResourceRoot.callerPath, + canonical_path: this.bundleResourceRoot.canonicalPath, + info_plist_path: this.bundleResourceRoot.infoPlistPath, + layout: this.bundleResourceRoot.layout, + executable_sha256: this.bundleResourceRoot.executableSha256, + }, + }), provenance, limitations: parsed.value.limitations, }); diff --git a/src/native/NativePlistResourceRoot.test.ts b/src/native/NativePlistResourceRoot.test.ts new file mode 100644 index 0000000000..81ef2beb38 --- /dev/null +++ b/src/native/NativePlistResourceRoot.test.ts @@ -0,0 +1,46 @@ +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { NativeMacOSProvider } from "./NativeMacOSProvider.js"; +import { resolveBundleResourcePlistSelection } from "./NativePlistSource.js"; +import { NativeFixtureRunner } from "../../tests/fixtures/nativeCommands.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { writeFile } from "node:fs/promises"; + +it("inspects a resolved app plist without executable identity", async () => { + const bundle = await createTestTempDirectory("rea-plist-resource-root-"); + const infoPlist = join(bundle, "Info.plist"); + await writeFile(infoPlist, ''); + const root = { + callerPath: bundle, + canonicalPath: bundle, + infoPlistPath: infoPlist, + layout: "root" as const, + executableSha256: null, + }; + expect(resolveBundleResourcePlistSelection(root, undefined)).toEqual({ + ok: true, + value: { path: infoPlist, origin: "default" }, + }); + expect(resolveBundleResourcePlistSelection(root, "Info.plist")).toEqual({ + ok: true, + value: { path: infoPlist, origin: "selected" }, + }); + const client = new NativeMacOSProvider( + {}, + new NativeFixtureRunner(), + "darwin", + ).createBundleResourceClient(root); + onTestFinished(async () => { + await client.close(); + }); + const inspected = await client.execute("inspect_plist", {}); + expect(inspected.ok).toBe(true); + if (inspected.ok) + expect(inspected.value.result).toMatchObject({ + source_path: infoPlist, + resource_root: { + canonical_path: bundle, + executable_sha256: null, + }, + }); +}); diff --git a/src/native/NativePlistSource.ts b/src/native/NativePlistSource.ts index 4c4d38d014..8c6f2e5828 100644 --- a/src/native/NativePlistSource.ts +++ b/src/native/NativePlistSource.ts @@ -4,6 +4,7 @@ import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import type { FileHandle } from "node:fs/promises"; import type { z } from "zod"; import type { inspectPlistSchema } from "../domain/native/nativeInspection.js"; +import type { ResolvedAppBundleResourceRoot } from "../application/AppBundleExecutable.js"; import type { AnalysisError, AnalysisErrorOptions, @@ -68,6 +69,21 @@ export const resolveNativePlistSelection = async ( } }; +/** Resolve a plist selection from the separately identified app resource root. */ +export const resolveBundleResourcePlistSelection = ( + root: ResolvedAppBundleResourceRoot, + requested: string | undefined, +): Result => { + if (requested === undefined) + return ok({ path: root.infoPlistPath, origin: "default" }); + return ok({ + path: isAbsolute(requested) + ? resolve(requested) + : resolve(root.canonicalPath, requested), + origin: "selected", + }); +}; + /** The bundle whose Contents/MacOS directory directly holds the program. */ const macOSBundleRoot = (path: string): string | undefined => { const programs = dirname(path); diff --git a/src/providers/AnalysisClientCleanup.ts b/src/providers/AnalysisClientCleanup.ts index b926488f0c..6bb3057687 100644 --- a/src/providers/AnalysisClientCleanup.ts +++ b/src/providers/AnalysisClientCleanup.ts @@ -12,21 +12,25 @@ export const closeAnalysisClient = async ( providerId: string, options: Pick & { readonly retainDocument?: boolean; + readonly onCleanupFailure?: (failure: AnalysisError) => void; } = {}, ): Promise> => { try { - return await client.close(options); + const result = await client.close(options); + if (!result.ok) options.onCleanupFailure?.(result.error); + return result; } catch (cause: unknown) { // A typed cleanup failure already names its leftover resources. - if (cause instanceof AnalysisError && cause.cleanupIncomplete) - return err(cause); - return err( - new ProviderCleanupError( - providerId, - ["provider-client"], - { reason: "provider client close rejected unexpectedly" }, - { cause }, - ), - ); + const failure = + cause instanceof AnalysisError && cause.cleanupIncomplete + ? cause + : new ProviderCleanupError( + providerId, + ["provider-client"], + { reason: "provider client close rejected unexpectedly" }, + { cause }, + ); + options.onCleanupFailure?.(failure); + return err(failure); } }; diff --git a/src/server/BundleResourcesMcp.test.ts b/src/server/BundleResourcesMcp.test.ts new file mode 100644 index 0000000000..be852c1eaf --- /dev/null +++ b/src/server/BundleResourcesMcp.test.ts @@ -0,0 +1,428 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import type { AnySchema } from "ajv"; +import { afterEach, expect, it } from "vitest"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { z } from "zod"; + +import { + createAnalysisExecution, + type AnalysisClient, + type ProviderIdentity, +} from "../application/AnalysisProvider.js"; +import { InvestigationRecords } from "../application/investigation/InvestigationRecords.js"; +import { BinarySession } from "../application/binary/BinarySession.js"; +import { AnalysisProviderRegistry } from "../application/binary/AnalysisProviderRegistry.js"; +import { SessionProviderRouter } from "../application/binary/SessionProviderRouter.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import { createEvidence } from "../domain/evidence.js"; +import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; +import { createServer } from "./createServer.js"; +import { createArtifactBinarySession } from "../composition/binary.js"; +import { build } from "plist"; + +const identity: ProviderIdentity = { + id: "bundle-resource-mcp-fixture", + name: "Bundle resource MCP fixture", + version: "1", +}; + +const resources: Array<{ close(): Promise }> = []; + +afterEach(async () => { + await Promise.all(resources.splice(0).map((resource) => resource.close())); +}); + +it("routes bundle resource MCP calls with null executable identity and retained Evidence", async () => { + const parent = await mkdtemp(join(tmpdir(), "rea-bundle-resource-mcp-")); + const bundle = join(parent, "Fixture.app"); + await mkdir(bundle); + const plistPath = join(bundle, "Info.plist"); + await writeFile(plistPath, ''); + + const resourceClient: AnalysisClient = { + execute: async (operation) => { + if (operation === "health") + return { ok: true, value: createAnalysisExecution(null, identity) }; + return { + ok: true, + value: createAnalysisExecution( + { + format: "xml", + value: {}, + bundle: { + identifier: null, + executable: null, + name: null, + version: null, + short_version: null, + }, + source_path: plistPath, + resource_root: { + caller_path: bundle, + canonical_path: bundle, + info_plist_path: plistPath, + layout: "root", + executable_sha256: null, + }, + provenance: [], + limitations: [], + }, + identity, + { subject: null }, + ), + }; + }, + close: async () => ({ ok: true, value: null }), + }; + const session = new BinarySession( + SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []), + new InvestigationRecords(), + { createClient: () => ({ client: resourceClient, capabilities: [] }) }, + ); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "bundle-resource-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push({ + close: async () => { + await client.close(); + await server.close(); + await session.close(); + await rm(parent, { recursive: true, force: true }); + }, + }); + + await server.connect(serverTransport); + await client.connect(clientTransport); + + await expectResourceSchemaBranches(client); + + const opened = await client.callTool({ + name: "open_bundle_resources", + arguments: { path: bundle }, + }); + expect(opened.isError).not.toBe(true); + expect( + toolContract("open_bundle_resources").outputSchema.parse( + opened.structuredContent, + ), + ).toMatchObject({ + result: { + canonical_path: bundle, + executable_sha256: null, + }, + }); + + const status = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + expect(status.isError).not.toBe(true); + expect( + toolContract("binary_session").outputSchema.parse(status.structuredContent), + ).toMatchObject({ + result: { + open: true, + session_kind: "bundle_resources", + resource_root: { canonical_path: bundle, executable_sha256: null }, + }, + }); + + const inspected = await client.callTool({ + name: "inspect_plist", + arguments: { path: plistPath }, + }); + expect(inspected.isError).not.toBe(true); + const evidence = toolContract("inspect_plist").outputSchema.parse( + inspected.structuredContent, + ); + expect(evidence).toMatchObject({ + subject: null, + normalized_result: { + source_path: plistPath, + resource_root: { executable_sha256: null }, + }, + }); + const evidenceId = z + .object({ evidence_id: z.string() }) + .parse(evidence).evidence_id; + expect(session.evidenceById(evidenceId)).toMatchObject({ + subject: null, + operation: "inspect_plist", + normalized_result: { + resource_root: { canonical_path: bundle, executable_sha256: null }, + }, + }); +}); + +it("exposes the production resource session through the MCP SDK for a keyless bundle", async () => { + const parent = await mkdtemp( + join(tmpdir(), "rea-bundle-resource-production-mcp-"), + ); + const bundle = join(parent, "Fixture.app"); + const resourceDirectory = join(bundle, "Resources"); + await mkdir(resourceDirectory, { recursive: true }); + await writeFile(join(bundle, "Info.plist"), ""); + await writeFile( + join(resourceDirectory, "Archive.plist"), + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "resource-only-value"], + $top: { root: { UID: 1 } }, + }), + ); + + const session = await createArtifactBinarySession({}, process.platform); + const server = createServer({ kind: "session", session }); + const client = new Client({ + name: "production-bundle-resource-test", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push({ + close: async () => { + await client.close(); + await server.close(); + await session.close(); + await rm(parent, { recursive: true, force: true }); + }, + }); + await server.connect(serverTransport); + await client.connect(clientTransport); + + const opened = await client.callTool({ + name: "open_bundle_resources", + arguments: { path: bundle }, + }); + expect(opened.isError).not.toBe(true); + expect( + toolContract("open_bundle_resources").outputSchema.parse( + opened.structuredContent, + ), + ).toMatchObject({ + result: { canonical_path: bundle, executable_sha256: null }, + }); + + const decoded = await client.callTool({ + name: "decode_interface_builder", + arguments: {}, + }); + expect(decoded.isError).not.toBe(true); + const decodedEvidence = toolContract( + "decode_interface_builder", + ).outputSchema.parse(decoded.structuredContent); + expect(decodedEvidence).toMatchObject({ + subject: null, + normalized_result: { + target_sha256: null, + resource_root: { canonical_path: bundle, executable_sha256: null }, + }, + }); + const decodedEvidenceId = z + .object({ evidence_id: z.string() }) + .parse(decodedEvidence).evidence_id; + for (const view of [ + { kind: "summary" }, + { + kind: "interface-builder", + collection: "documents", + offset: 0, + limit: 64, + }, + ] as const) { + const projected = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: decodedEvidenceId }, + view, + }, + }); + expect(projected.isError).not.toBe(true); + expect( + toolContract("inspect_analysis_view").outputSchema.parse( + projected.structuredContent, + ), + ).toMatchObject({ + normalized_result: { + artifact: { path: bundle, sha256: null }, + resource_root: { + caller_path: bundle, + canonical_path: bundle, + executable_sha256: null, + }, + }, + }); + } + + const inspected = await client.callTool({ + name: "inspect_keyed_archive", + arguments: { path: "Resources/Archive.plist" }, + }); + expect(inspected.isError).not.toBe(true); + const evidence = toolContract("inspect_keyed_archive").outputSchema.parse( + inspected.structuredContent, + ); + expect(evidence).toMatchObject({ + subject: null, + normalized_result: { + archive_path: "Resources/Archive.plist", + target_sha256: null, + resource_root: { canonical_path: bundle, executable_sha256: null }, + objects: expect.arrayContaining([ + expect.objectContaining({ value: "resource-only-value" }), + ]), + }, + }); + const evidenceId = z + .object({ evidence_id: z.string() }) + .parse(evidence).evidence_id; + expect(session.evidenceById(evidenceId)).toMatchObject({ + subject: null, + operation: "inspect_keyed_archive", + normalized_result: { + resource_root: { canonical_path: bundle, executable_sha256: null }, + }, + }); +}); + +const expectResourceSchemaBranches = async (client: Client): Promise => { + const schemaCases: Array<{ + readonly name: + | "decode_interface_builder" + | "inspect_asset_catalog" + | "inspect_keyed_archive"; + readonly validResult: JsonValue; + }> = [ + { + name: "decode_interface_builder", + validResult: jsonObjectSchema.parse({ + target_sha256: "a".repeat(64), + documents: [], + graph: { + target_sha256: "a".repeat(64), + provider: { id: "fixture", version: null, tool_version: "1" }, + nodes: [], + edges: [], + coverage: [], + truncated: false, + }, + limitations: [], + }), + }, + { + name: "inspect_asset_catalog", + validResult: jsonObjectSchema.parse({ + target_sha256: "a".repeat(64), + catalogs: [], + total_records: 0, + offset: 0, + limit: 1, + records: [], + resource_key_matches: [], + next_offset: null, + truncated: false, + limitations: [], + }), + }, + { + name: "inspect_keyed_archive", + validResult: jsonObjectSchema.parse({ + target_sha256: "a".repeat(64), + archive_path: "Resources/data.archive", + archive_sha256: "b".repeat(64), + archive_format: "binary-plist", + roots: {}, + objects: [], + references: [], + total_objects: 0, + total_references: 0, + offset: 0, + next_offset: null, + truncated: false, + limitations: [], + }), + }, + ] as const; + const advertisedTools = new Map( + (await client.listTools()).tools.map((tool) => [tool.name, tool]), + ); + const resourceRoot = { + caller_path: "/tmp/Fixture.app", + canonical_path: "/tmp/Fixture.app", + info_plist_path: "/tmp/Fixture.app/Info.plist", + layout: "root", + executable_sha256: null, + } as const; + for (const { name, validResult } of schemaCases) { + const contract = toolContract(name); + const validEvidence = createEvidence(undefined, identity, { + operation: name, + parameters: {}, + result: validResult, + }); + expect(contract.outputSchema.safeParse(validEvidence).success, name).toBe( + true, + ); + const malformedResult = structuredClone(validResult) as Record< + string, + JsonValue + >; + malformedResult.target_sha256 = null; + if (name === "decode_interface_builder") { + malformedResult.graph = { + ...(malformedResult.graph as Record), + target_sha256: null, + }; + } + const malformedEvidence = createEvidence(undefined, identity, { + operation: name, + parameters: {}, + result: malformedResult, + }); + expect( + contract.outputSchema.safeParse(malformedEvidence).success, + `${name} canonical rejects missing resource_root`, + ).toBe(false); + const advertised = advertisedTools.get(name)?.outputSchema; + expect(advertised, `${name} is advertised`).toBeDefined(); + if (advertised === undefined) continue; + const validateAdvertised = new Ajv2020({ + strict: false, + validateFormats: false, + }).compile(advertised as AnySchema); + const resourceResult = structuredClone(validResult) as Record< + string, + JsonValue + >; + resourceResult.target_sha256 = null; + resourceResult.resource_root = resourceRoot; + if (name === "decode_interface_builder") { + resourceResult.graph = { + ...(resourceResult.graph as Record), + target_sha256: null, + }; + } + const resourceEvidence = createEvidence(undefined, identity, { + operation: name, + parameters: {}, + result: resourceResult, + }); + expect( + contract.outputSchema.safeParse(resourceEvidence).success, + name, + ).toBe(true); + expect( + validateAdvertised(resourceEvidence), + `${name} advertised root branch`, + ).toBe(true); + expect( + validateAdvertised(malformedEvidence), + `${name} advertised schema rejects missing resource_root`, + ).toBe(false); + } +}; diff --git a/src/server/registerEvidenceTools.ts b/src/server/registerEvidenceTools.ts index 8fc3867d64..6d87921842 100644 --- a/src/server/registerEvidenceTools.ts +++ b/src/server/registerEvidenceTools.ts @@ -107,7 +107,9 @@ export const registerEvidenceTools = ( options.sourceEvidence?.(contract.name, execution.value.result) ?? []; const evidence = createEvidence( - execution.value.subject ?? options.activeTarget?.(), + execution.value.subject === undefined + ? options.activeTarget?.() + : (execution.value.subject ?? undefined), execution.value.provider, { operation: contract.name, diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index 68c6ddbaa6..4bc6ac6bb8 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -278,6 +278,7 @@ export const registerSessionTools = ( options: SessionToolOptions, ): void => { const openContract = toolContract("open_binary"); + const openBundleResourcesContract = toolContract("open_bundle_resources"); const closeContract = toolContract("close_binary"); const statusContract = toolContract("binary_session"); const exportContract = toolContract("export_evidence_bundle"); @@ -289,6 +290,39 @@ export const registerSessionTools = ( const compareBundlesContract = toolContract("compare_bundles"); const snapshotContract = toolContract("get_evidence_bundle"); const processResources = new ProcessCaptureResourceScope(); + server.registerTool( + openBundleResourcesContract.name, + toolRegistrationOptions(openBundleResourcesContract), + async (input, context) => { + const opened = await logToolExecution( + logger, + openBundleResourcesContract.name, + () => + session.openBundleResources(input.path, { + signal: context.mcpReq.signal, + }), + ); + if (!opened.ok) + return server.delivery.toCallToolResult( + opened, + openBundleResourcesContract, + ); + const root = opened.value; + return server.delivery.toCallToolResult( + { + ok: true, + value: { + caller_path: root.callerPath, + canonical_path: root.canonicalPath, + info_plist_path: root.infoPlistPath, + layout: root.layout, + executable_sha256: root.executableSha256, + }, + }, + openBundleResourcesContract, + ); + }, + ); registerLifecycleTools( { ...options, diff --git a/tests/acceptance/applications/bundleResourceRootCliMcp.test.ts b/tests/acceptance/applications/bundleResourceRootCliMcp.test.ts new file mode 100644 index 0000000000..00539fd2e6 --- /dev/null +++ b/tests/acceptance/applications/bundleResourceRootCliMcp.test.ts @@ -0,0 +1,234 @@ +import { build } from "plist"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const resourceOperations = [ + "inspect_plist", + "decode_interface_builder", + "inspect_asset_catalog", + "inspect_keyed_archive", +] as const; + +cliTest.skipIf(process.platform !== "darwin")( + "inspects a keyless app resource root through the compiled CLI", + async ({ cli }) => { + const temporaryRoot = await createTestTempDirectory( + "rea-bundle-resource-acceptance-", + ); + const firstBundle = await createKeylessBundle(temporaryRoot, "First"); + + for (const scenario of cliScenarios(firstBundle)) { + const response = await cli.run({ + arguments: [...scenario.arguments, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect( + response.exitCode, + `${scenario.operation}: ${response.stdout} ${response.stderr}`, + ).toBe(0); + const evidence = expectResourceEvidence( + response.json, + scenario.operation, + firstBundle, + ); + if (scenario.operation === "inspect_keyed_archive") + expect(evidence.normalized_result).toMatchObject({ + archive_path: "Resources/Archive.plist", + objects: expect.arrayContaining([ + expect.objectContaining({ value: "resource-only-value" }), + ]), + }); + } + }, +); + +cliTest.skipIf(process.platform !== "darwin")( + "retains and switches keyless resource sessions through stdio MCP", + async () => { + const temporaryRoot = await createTestTempDirectory( + "rea-resource-session-acceptance-", + ); + const firstBundle = await createKeylessBundle(temporaryRoot, "First"); + const secondBundle = await createKeylessBundle(temporaryRoot, "Second"); + const { call } = await connectLocalToolsMcp(); + const opened = await call("open_bundle_resources", { path: firstBundle }); + expect(opened.isError).not.toBe(true); + expect( + toolContract("open_bundle_resources").outputSchema.parse( + opened.structuredContent, + ), + ).toMatchObject({ + result: { + caller_path: firstBundle, + canonical_path: firstBundle, + executable_sha256: null, + }, + }); + + const firstStatus = await call("binary_session", {}); + expect(firstStatus.isError).not.toBe(true); + expect( + toolContract("binary_session").outputSchema.parse( + firstStatus.structuredContent, + ), + ).toMatchObject({ + result: { + open: true, + session_kind: "bundle_resources", + analysis_provider_binding: null, + analysis_provider_candidates: [], + resource_root: { + canonical_path: firstBundle, + executable_sha256: null, + }, + }, + }); + + for (const operation of resourceOperations) { + const arguments_ = + operation === "inspect_keyed_archive" + ? { path: "Resources/Archive.plist" } + : {}; + const response = await call(operation, arguments_); + expect(response.isError, operation).not.toBe(true); + const evidence = expectResourceEvidence( + response.structuredContent, + operation, + firstBundle, + ); + if (operation === "inspect_keyed_archive") + expect(evidence.normalized_result).toMatchObject({ + archive_path: "Resources/Archive.plist", + objects: expect.arrayContaining([ + expect.objectContaining({ value: "resource-only-value" }), + ]), + }); + } + + const retained = await call("get_evidence_bundle", {}); + expect(retained.isError).not.toBe(true); + const evidenceBundle = toolContract( + "get_evidence_bundle", + ).outputSchema.parse(retained.structuredContent); + for (const operation of resourceOperations) { + const record = evidenceBundle.result.records.find( + (record) => record.operation === operation, + ); + expectResourceEvidence(record, operation, firstBundle); + } + + const switched = await call("open_bundle_resources", { + path: secondBundle, + }); + expect(switched.isError).not.toBe(true); + expect( + toolContract("open_bundle_resources").outputSchema.parse( + switched.structuredContent, + ), + ).toMatchObject({ result: { canonical_path: secondBundle } }); + const switchedStatus = await call("binary_session", {}); + expect(switchedStatus.isError).not.toBe(true); + expect( + toolContract("binary_session").outputSchema.parse( + switchedStatus.structuredContent, + ), + ).toMatchObject({ + result: { + open: true, + session_kind: "bundle_resources", + resource_root: { + caller_path: secondBundle, + canonical_path: secondBundle, + executable_sha256: null, + }, + }, + }); + + const closed = await call("close_binary", {}); + expect(closed.isError).not.toBe(true); + toolContract("close_binary").outputSchema.parse(closed.structuredContent); + const finalStatus = await call("binary_session", {}); + expect(finalStatus.isError).not.toBe(true); + expect( + toolContract("binary_session").outputSchema.parse( + finalStatus.structuredContent, + ), + ).toMatchObject({ result: { open: false, cleanup_pending: false } }); + }, +); + +const createKeylessBundle = async ( + root: string, + name: string, +): Promise => { + const bundle = join(root, `${name}.app`); + await mkdir(join(bundle, "Resources"), { recursive: true }); + await mkdir(join(bundle, "Assets.xcassets")); + await writeFile( + join(bundle, "Info.plist"), + build({ CFBundleIdentifier: `org.example.${name.toLowerCase()}` }), + ); + await writeFile( + join(bundle, "Resources", "Archive.plist"), + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "resource-only-value"], + $top: { root: { UID: 1 } }, + }), + ); + return bundle; +}; + +const cliScenarios = (bundle: string) => + [ + { operation: "inspect_plist", arguments: ["inspect-plist", bundle] }, + { + operation: "decode_interface_builder", + arguments: ["decode-interface-builder", bundle], + }, + { + operation: "inspect_asset_catalog", + arguments: ["inspect-asset-catalog", bundle], + }, + { + operation: "inspect_keyed_archive", + arguments: ["inspect-keyed-archive", bundle, "Resources/Archive.plist"], + }, + ] as const; + +const expectResourceEvidence = ( + value: unknown, + operation: (typeof resourceOperations)[number], + bundle: string, +) => { + const evidence = parseEvidence( + toolContract(operation).outputSchema.parse(value), + ); + expect(evidence).toMatchObject({ + operation, + subject: null, + normalized_result: { + resource_root: { + caller_path: bundle, + canonical_path: bundle, + executable_sha256: null, + }, + }, + }); + if (operation === "inspect_plist") + expect(evidence.normalized_result).toMatchObject({ + source_path: join(bundle, "Info.plist"), + bundle: { executable: null }, + }); + else + expect(evidence.normalized_result).toMatchObject({ target_sha256: null }); + return evidence; +}; diff --git a/tests/boundary/providers/bundleResourceComposition.test.ts b/tests/boundary/providers/bundleResourceComposition.test.ts new file mode 100644 index 0000000000..448c04e1df --- /dev/null +++ b/tests/boundary/providers/bundleResourceComposition.test.ts @@ -0,0 +1,115 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { build } from "plist"; +import { afterEach, expect, it } from "vitest"; + +import { parseConfig } from "../../../src/config/parseConfig.js"; +import type { DirectAnalysisDependencies } from "../../../src/application/DirectAnalysisDependencies.js"; +import { runProviderAnalysis } from "../../../src/application/DirectAnalysis.js"; +import { + createArtifactBinarySession, + createBinarySession, + createManagedBinarySession, + createNativeBinarySession, + createNativeDispatchBinarySession, +} from "../../../src/composition/binary.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +it("routes a CLI app resource path through the resource session and preserves null target identity", async () => { + const bundlePath = await mkdtemp(join(tmpdir(), "rea-cli-resource-root-")); + temporaryDirectories.push(bundlePath); + await mkdir(join(bundlePath, "Resources")); + await writeFile( + join(bundlePath, "Info.plist"), + '', + ); + await writeFile( + join(bundlePath, "Resources", "Archive.plist"), + build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "sample"], + $top: { root: { UID: 1 } }, + }), + ); + const dependencies: DirectAnalysisDependencies = { + readConfiguration: () => parseConfig({}), + createBinarySession: (config, logger) => + createBinarySession(config, logger, {}), + createNativeDispatchBinarySession: (config, logger) => + createNativeDispatchBinarySession(config, logger, {}), + createArtifactBinarySession: () => + createArtifactBinarySession({}, process.platform), + createNativeBinarySession: () => + createNativeBinarySession({}, process.platform), + createManagedBinarySession, + }; + const result = await runProviderAnalysis( + dependencies, + bundlePath, + "inspect_keyed_archive", + { path: "Resources/Archive.plist" }, + ); + expect(result).toMatchObject({ + operation: "inspect_keyed_archive", + subject: null, + normalized_result: { + target_sha256: null, + resource_root: { canonical_path: bundlePath, executable_sha256: null }, + }, + }); +}); + +it("reports resource capabilities from their providers on supported and unsupported hosts", async () => { + const rootPath = await mkdtemp(join(tmpdir(), "rea-resource-capabilities-")); + temporaryDirectories.push(rootPath); + await writeFile(join(rootPath, "Info.plist"), ""); + + for (const [platform, expected] of [ + ["darwin", { plist: true, assets: true }], + ["linux", { plist: false, assets: false }], + ] as const) { + const session = await createArtifactBinarySession({}, platform); + const opened = await session.openBundleResources(rootPath); + expect(opened.ok, platform).toBe(true); + const status = await session.status(); + const capabilitySets = ( + status as { + capabilities: Array<{ + groups: Array<{ operations: string[]; available: boolean }>; + }>; + } + ).capabilities; + const operationAvailability = new Map( + capabilitySets.flatMap(({ groups }) => + groups.flatMap(({ operations, available }) => + operations.map((operation) => [operation, available] as const), + ), + ), + ); + expect(operationAvailability.get("inspect_plist"), platform).toBe( + expected.plist, + ); + expect(operationAvailability.get("inspect_asset_catalog"), platform).toBe( + expected.assets, + ); + expect( + operationAvailability.get("decode_interface_builder"), + platform, + ).toBe(true); + expect(operationAvailability.get("inspect_keyed_archive"), platform).toBe( + true, + ); + await session.close(); + } +}); From c4b6aa3061d82c81c9172e242ea1871ad52cc6f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulises=20Mill=C3=A1n?= Date: Tue, 20 Oct 2026 14:35:45 -0600 Subject: [PATCH 12/35] fix(managed): treat extension-only module scopes as unknown identity (#5674) Closes #5467 --- .../managed/managedNativeVerificationMatch.ts | 16 +++++++++++----- .../managed/managedNativeVerification.test.ts | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/domain/managed/managedNativeVerificationMatch.ts b/src/domain/managed/managedNativeVerificationMatch.ts index ef7c817473..9bccc56e85 100644 --- a/src/domain/managed/managedNativeVerificationMatch.ts +++ b/src/domain/managed/managedNativeVerificationMatch.ts @@ -113,12 +113,20 @@ const normalizeModule = (value: string): string => .toLowerCase() .replace(/\.(dll|dylib|so|node|exe)$/u, ""); +/** A declared module identity, or null when normalization leaves no name. */ +const declaredModuleIdentity = (scope: string | null): string | null => { + if (scope === null || scope.trim().length === 0) return null; + const identity = normalizeModule(scope); + return identity.length > 0 ? identity : null; +}; + const nativeModuleIdentities = (value: string): ReadonlySet => { const filename = basename(value).toLowerCase(); const stem = normalizeModule(filename); const identities = new Set([stem]); if (/\.(dylib|so)$/u.test(filename) && stem.startsWith("lib")) identities.add(stem.slice("lib".length)); + identities.delete(""); return identities; }; @@ -126,8 +134,8 @@ const moduleCompatible = ( scope: string | null, symbol: NativeSymbol, ): boolean => { - if (scope === null || scope.trim().length === 0) return false; - const expected = normalizeModule(scope); + const expected = declaredModuleIdentity(scope); + if (expected === null) return false; return [symbol.module_name, symbol.module_path] .filter( (value): value is string => @@ -200,9 +208,7 @@ const matchPinvoke = ( }; const candidates: [NativeSymbol, ...NativeSymbol[]] = [first, ...remaining]; - const scopeKnown = - managed.import_scope_name !== null && - managed.import_scope_name.trim().length > 0; + const scopeKnown = declaredModuleIdentity(managed.import_scope_name) !== null; const exact = candidates.find( (symbol) => managed.import_name === normalizedSymbolName(symbol) && diff --git a/tests/conformance/providers/managed/managedNativeVerification.test.ts b/tests/conformance/providers/managed/managedNativeVerification.test.ts index b15c720fd6..bd39d4b277 100644 --- a/tests/conformance/providers/managed/managedNativeVerification.test.ts +++ b/tests/conformance/providers/managed/managedNativeVerification.test.ts @@ -263,6 +263,22 @@ it("keeps name candidates unresolved when a required module identity is absent", ); } + for (const path of [ + "/examples/lib.dylib", + "/examples/lib.so", + "/examples/foo.dll", + ]) { + const extensionOnly = verifyManagedNativeBoundaries({ + ...inputWithImportScope(".dll"), + native_observations: [nativeEvidenceWithExports(["open_native"], path)], + }); + expect(extensionOnly.pinvoke_imports[0]).toMatchObject({ + status: "unresolved", + basis: "module-identity-unknown", + candidates: [{ name: "open_native", module_path: path }], + }); + } + const subjectless = verifyManagedNativeBoundaries({ ...inputWithImportScope("nativehelper.dll"), native_observations: [ From c38f96fa693de115a47498d467169b8c69f0a426 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:48:33 +0800 Subject: [PATCH 13/35] test(cli): keep IPC trace input under the Linux spawn limit (#5673) * test(cli): keep IPC trace input under the Linux spawn limit The listener-callsite CLI builds one trace argument from the full application Evidence document. On 4 KiB-page Linux that string exceeds MAX_ARG_STRLEN, so spawn throws E2BIG before the command starts. Keep the document in a file, require the spawned argv to fit the single-argument budget, and lock that budget through the test process registry. Co-authored-by: morluto * test(process): size the spawn ceiling from ARG_MAX and MAX_ARG_STRLEN A payload at the single-string limit still fails execve with E2BIG when the stack shrinks ARG_MAX below the rest of the image. Accept only the bytes that fit both budgets, and skip that success spawn when the fixed argv and environment already fill ARG_MAX. Co-authored-by: morluto * style(test): format files rejected by oxfmt in static checks check:ci:fast failed format:check on the CLI fixture import and, after merging main, on the admission test's new command-abort case. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- .../electronIpcListenerCallsiteCli.test.ts | 24 ++- .../process/spawnArgumentLimit.test.ts | 181 ++++++++++++++++++ .../reverseEngineeringAdmission.test.ts | 8 +- tests/support/cli/cliFixture.ts | 5 +- tests/support/process/spawnArgumentLimit.ts | 66 +++++++ 5 files changed, 273 insertions(+), 11 deletions(-) create mode 100644 tests/boundary/process/spawnArgumentLimit.test.ts create mode 100644 tests/support/process/spawnArgumentLimit.ts diff --git a/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts b/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts index 54f6883e5d..472603cf71 100644 --- a/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts +++ b/tests/acceptance/applications/electronIpcListenerCallsiteCli.test.ts @@ -9,6 +9,10 @@ import { applicationFeatureTraceResultSchema } from "../../../src/domain/javascr import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +import { + assertSpawnArgumentsWithinLimit, + LINUX_4K_SINGLE_ARGUMENT_BYTES, +} from "../../support/process/spawnArgumentLimit.js"; cliTest.for(["CLI", "stdio MCP"] as const)( "counts exposed IPC listener callsites once and retains bridge paths through %s", @@ -104,12 +108,24 @@ cliTest.for(["CLI", "stdio MCP"] as const)( }; let traceEvidence: unknown; if (surface === "CLI") { - // The application Evidence document exceeds the runner argument limit - // when it is placed on argv. The command reads the same JSON from a file. + // Linux counts one argv string plus its trailing NUL against + // MAX_ARG_STRLEN (32 pages) and returns E2BIG before the CLI starts. + // This Evidence document is past that ceiling, so the command reads the + // same JSON from a file instead of argv. + const serialized = JSON.stringify(traceArguments); + expect(Buffer.byteLength(serialized)).toBeGreaterThan( + LINUX_4K_SINGLE_ARGUMENT_BYTES, + ); const traceInput = join(root, "trace-input.json"); - await writeFile(traceInput, JSON.stringify(traceArguments)); + await writeFile(traceInput, serialized); + const traceCommand = [ + "trace-application-feature", + traceInput, + "--json", + ] as const; + assertSpawnArgumentsWithinLimit(traceCommand); const response = await cli.run({ - arguments: ["trace-application-feature", traceInput, "--json"], + arguments: traceCommand, }); expect(response.exitCode, response.stderr).toBe(0); traceEvidence = response.json; diff --git a/tests/boundary/process/spawnArgumentLimit.test.ts b/tests/boundary/process/spawnArgumentLimit.test.ts new file mode 100644 index 0000000000..78e64518ac --- /dev/null +++ b/tests/boundary/process/spawnArgumentLimit.test.ts @@ -0,0 +1,181 @@ +import { execFileSync } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { processTest } from "../../support/process/processFixture.js"; +import { + acceptedSpawnPayloadBytes, + assertSpawnArgumentsWithinLimit, + LINUX_4K_SINGLE_ARGUMENT_BYTES, +} from "../../support/process/spawnArgumentLimit.js"; + +const minimalSpawnEnvironment = (): NodeJS.ProcessEnv => ({ + PATH: process.env.PATH ?? "", + ...(process.platform === "win32" + ? { + SystemRoot: process.env.SystemRoot, + TEMP: tmpdir(), + TMP: tmpdir(), + } + : {}), +}); + +const linuxGetconf = (name: "PAGE_SIZE" | "ARG_MAX"): number => { + const raw = execFileSync("getconf", [name], { encoding: "utf8" }).trim(); + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < 1) + throw new Error(`unexpected ${name} ${raw}`); + return value; +}; + +/** Linux `execve` stores one native pointer per argv and environment string. */ +const linuxPointerBytes = (): number => + process.arch === "arm" || + process.arch === "ia32" || + process.arch === "mips" || + process.arch === "mipsel" + ? 4 + : 8; + +const execStringBytes = (value: string): number => Buffer.byteLength(value) + 1; + +const captureSpawnFailure = async ( + run: () => Promise, +): Promise => { + try { + await run(); + } catch (error) { + return error; + } + return undefined; +}; + +processTest( + "spawns a file path when the payload exceeds the single-argument budget", + async ({ processes }) => { + const root = await createTestTempDirectory("rea-spawn-argument-"); + const payload = "x".repeat(LINUX_4K_SINGLE_ARGUMENT_BYTES + 1); + const file = join(root, "payload.txt"); + await writeFile(file, payload); + expect(() => assertSpawnArgumentsWithinLimit([file])).not.toThrow(); + expect(() => assertSpawnArgumentsWithinLimit([payload])).toThrow(/E2BIG/u); + const result = await processes.run( + process.execPath, + ["-e", "process.exit(0)", file], + { env: minimalSpawnEnvironment(), timeoutMs: 15_000 }, + ); + expect(result.exitCode).toBe(0); + }, +); + +it("fits the accepted payload to both the single-string and aggregate budgets", () => { + const fixedStringBytes = 500; + const pointerCount = 5; + const pointerBytes = 8; + expect( + acceptedSpawnPayloadBytes({ + pageSize: 4096, + argMax: 2 * 1024 * 1024, + pointerBytes, + pointerCount, + fixedStringBytes, + }), + ).toBe(LINUX_4K_SINGLE_ARGUMENT_BYTES); + const tight = acceptedSpawnPayloadBytes({ + pageSize: 4096, + argMax: 32 * 4096, + pointerBytes, + pointerCount, + fixedStringBytes, + }); + expect(tight).toBe( + 32 * 4096 - pointerCount * pointerBytes - fixedStringBytes - 1, + ); + expect(tight).toBeLessThan(LINUX_4K_SINGLE_ARGUMENT_BYTES); + expect( + acceptedSpawnPayloadBytes({ + pageSize: 4096, + argMax: 32 * 4096, + pointerBytes, + pointerCount, + fixedStringBytes: 32 * 4096, + }), + ).toBeUndefined(); +}); + +it("counts UTF-8 bytes when applying the single-argument budget", () => { + const argument = "é".repeat( + Math.ceil((LINUX_4K_SINGLE_ARGUMENT_BYTES + 1) / 2), + ); + expect(argument.length).toBeLessThanOrEqual(LINUX_4K_SINGLE_ARGUMENT_BYTES); + expect(Buffer.byteLength(argument)).toBeGreaterThan( + LINUX_4K_SINGLE_ARGUMENT_BYTES, + ); + expect(() => assertSpawnArgumentsWithinLimit([argument])).toThrow(/E2BIG/u); + expect(() => + assertSpawnArgumentsWithinLimit([ + "x".repeat(LINUX_4K_SINGLE_ARGUMENT_BYTES), + ]), + ).not.toThrow(); +}); + +processTest.skipIf(process.platform !== "linux")( + "matches the kernel single-argument ceiling through the test process registry", + async ({ processes }) => { + const script = "process.exit(0)"; + const environment = minimalSpawnEnvironment(); + const environmentEntries = Object.entries(environment).flatMap( + ([key, value]) => + typeof value === "string" ? [[key, value] as const] : [], + ); + const pageSize = linuxGetconf("PAGE_SIZE"); + const singleStringBytes = 32 * pageSize - 1; + expect(LINUX_4K_SINGLE_ARGUMENT_BYTES).toBeLessThanOrEqual( + singleStringBytes, + ); + if (pageSize === 4096) + expect(LINUX_4K_SINGLE_ARGUMENT_BYTES).toBe(singleStringBytes); + // The filename is copied before argv, so the executable path is counted twice. + const fixedStringBytes = + execStringBytes(process.execPath) + + execStringBytes(process.execPath) + + execStringBytes("-e") + + execStringBytes(script) + + environmentEntries.reduce( + (sum, [key, value]) => sum + execStringBytes(`${key}=${value}`), + 0, + ); + const accepted = acceptedSpawnPayloadBytes({ + pageSize, + argMax: linuxGetconf("ARG_MAX"), + pointerBytes: linuxPointerBytes(), + pointerCount: Math.max(4, 1) + environmentEntries.length, + fixedStringBytes, + }); + + const run = (payloadBytes: number) => + processes.run( + process.execPath, + ["-e", script, "x".repeat(payloadBytes)], + { env: environment, timeoutMs: 15_000 }, + ); + if (accepted !== undefined) { + const result = await run(accepted); + expect(result.exitCode).toBe(0); + } + expect( + await captureSpawnFailure(() => run(singleStringBytes + 1)), + ).toMatchObject({ + code: "E2BIG", + }); + if (accepted !== undefined && accepted < singleStringBytes) { + expect(await captureSpawnFailure(() => run(accepted + 1))).toMatchObject({ + code: "E2BIG", + }); + } + }, +); diff --git a/tests/composition/reverseEngineeringAdmission.test.ts b/tests/composition/reverseEngineeringAdmission.test.ts index 8440827b34..cb4d03f9db 100644 --- a/tests/composition/reverseEngineeringAdmission.test.ts +++ b/tests/composition/reverseEngineeringAdmission.test.ts @@ -263,7 +263,9 @@ it.each([ ] as const)( "classifies a $code command failure with captured output $captured while the caller is aborted", async ({ captured, code, tag }) => { - const directory = await mkdtemp(join(tmpdir(), "rea-reverse-command-abort-")); + const directory = await mkdtemp( + join(tmpdir(), "rea-reverse-command-abort-"), + ); const path = join(directory, "fixture.bin"); await writeFile(path, "fixture"); const reason = Object.freeze({ requested: `stop-${code}` }); @@ -287,9 +289,7 @@ it.each([ ); expect(result).toMatchObject({ ok: false, error: { _tag: tag } }); if (result.ok) return; - expect(result.error.cause).toBe( - code === "ABORT_ERR" ? reason : failure, - ); + expect(result.error.cause).toBe(code === "ABORT_ERR" ? reason : failure); if (captured && code === "ABORT_ERR") { expect(result.error.capturedOutput).toMatchObject({ stdout: "partial stdout", diff --git a/tests/support/cli/cliFixture.ts b/tests/support/cli/cliFixture.ts index 3ee43177a5..9d093c7da4 100644 --- a/tests/support/cli/cliFixture.ts +++ b/tests/support/cli/cliFixture.ts @@ -182,9 +182,8 @@ const createFileMcpSession = ( compileCache: string, ): { session: SharedMcpSession; close(): Promise } => createSharedMcpSession(async (registerCleanup, isClosed) => { - const { connectLocalToolsMcp } = await import( - "../../fixtures/localToolsMcp.js" - ); + const { connectLocalToolsMcp } = + await import("../../fixtures/localToolsMcp.js"); if (isClosed()) throw sharedMcpSessionClosed(); const mcp = await connectLocalToolsMcp( { NODE_COMPILE_CACHE: compileCache }, diff --git a/tests/support/process/spawnArgumentLimit.ts b/tests/support/process/spawnArgumentLimit.ts new file mode 100644 index 0000000000..962d934cfe --- /dev/null +++ b/tests/support/process/spawnArgumentLimit.ts @@ -0,0 +1,66 @@ +import { Buffer } from "node:buffer"; + +/** + * Largest single argv or environment string Linux will pass to `execve` on a + * 4 KiB page host. The kernel's `MAX_ARG_STRLEN` is 32 pages and includes the + * trailing NUL, so the caller-visible budget is one byte smaller. GitHub + * Actions runners use 4 KiB pages. One argument past this budget fails with + * `E2BIG` before the process starts. The combined argv, environment, filename, + * and pointer table must also fit in `ARG_MAX`, which shrinks with the stack. + */ +export const LINUX_4K_SINGLE_ARGUMENT_BYTES = 32 * 4096 - 1; + +/** + * Largest payload that fits this execve image. + * + * `fixedStringBytes` counts the filename, every argv string except the + * payload, and every `KEY=value` environment string, each including its NUL. + * `pointerCount` is `max(argc, 1) + envc`: Linux reserves that many native + * pointers inside `ARG_MAX` before it copies strings. The payload's own NUL + * is reserved here. The result is undefined when the fixed image already + * fills `ARG_MAX`. + */ +export const acceptedSpawnPayloadBytes = (input: { + readonly pageSize: number; + readonly argMax: number; + readonly pointerBytes: number; + readonly pointerCount: number; + readonly fixedStringBytes: number; +}): number | undefined => { + for (const [name, value] of [ + ["pageSize", input.pageSize], + ["argMax", input.argMax], + ["pointerBytes", input.pointerBytes], + ["pointerCount", input.pointerCount], + ["fixedStringBytes", input.fixedStringBytes], + ] as const) { + if (!Number.isSafeInteger(value) || value < 0) + throw new Error(`${name} must be a non-negative safe integer`); + } + if (input.pageSize < 1 || input.pointerBytes < 1 || input.pointerCount < 1) + throw new Error( + "page size, pointer size, and pointer count must be positive", + ); + const singleStringBytes = 32 * input.pageSize - 1; + const aggregatePayload = + input.argMax - + input.pointerCount * input.pointerBytes - + input.fixedStringBytes - + 1; + if (aggregatePayload < 1) return undefined; + return Math.min(singleStringBytes, aggregatePayload); +}; + +/** Require every argv string to fit in the Linux single-argument budget. */ +export const assertSpawnArgumentsWithinLimit = ( + arguments_: readonly string[], +): void => { + for (const argument of arguments_) { + const bytes = Buffer.byteLength(argument); + if (bytes > LINUX_4K_SINGLE_ARGUMENT_BYTES) { + throw new Error( + `spawn argument is ${bytes} bytes; one Linux argument above ${LINUX_4K_SINGLE_ARGUMENT_BYTES} bytes fails with E2BIG before the process starts. Pass the payload through a file.`, + ); + } + } +}; From 0bc2b42001f684beffcf47b70e507443b1a5d320 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:00:37 +0800 Subject: [PATCH 14/35] fix(browser)!: preserve unknown WebSocket frame facts Keep malformed or missing CDP opcodes and payload facts unknown, retain observed zero and empty payloads, and avoid false capture differences while preserving independently known facts. Validation: 59 source tests; compiled public CLI and stdio MCP fake-CDP acceptance; check:fast; docs:check; changed-file formatting. Real Chromium malformed-event behavior was not tested. BREAKING CHANGE: WebSocket event opcode is nullable and numeric values are restricted to the four-bit protocol range. Fixes #5666 --- docs/browser-observation.md | 2 +- src/browser/CdpCaptureEventHandlers.ts | 31 ++- .../CdpCaptureEvents.webSocketFrames.test.ts | 182 ++++++++++++++++++ src/domain/browserObservationSchemas.ts | 2 +- src/domain/webCaptureDiff.network.test.ts | 68 +++++++ src/domain/webCaptureDiff.ts | 51 +++-- .../browser/webSocketOpcodeCliMcp.test.ts | 148 ++++++++++++++ 7 files changed, 458 insertions(+), 26 deletions(-) create mode 100644 src/browser/CdpCaptureEvents.webSocketFrames.test.ts create mode 100644 tests/acceptance/browser/webSocketOpcodeCliMcp.test.ts diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 6034644493..305de00638 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -362,7 +362,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex WebSocket coverage. Redirect resource types use only shared known metadata: an omitted type leaves equality unknown rather than proving a modification. - Console observations with a stack source retain call type, argument types, the CDP `Runtime.Timestamp` in milliseconds since Unix epoch, and source location. An absent or invalid timestamp is `null` and makes `console_events` incomplete; an observed zero remains zero. This epoch-millisecond clock is separate from the monotonic-second timestamps in `observe-web-session`. When console text capture is selected, already-delivered primitive values are retained verbatim; objects, getters, and remote properties are never expanded. -- WebSocket observations retain direction, opcode, and payload byte length. When shape capture is selected, text frames are classified as text or complete value-free JSON shape; binary bytes, hashes, prefixes, and raw frames are never retained. +- WebSocket observations retain direction, opcode, and payload byte length. A missing or malformed CDP opcode is returned as `null`, with opcode-dependent payload byte length and shape also `null`, and marks `websocket_frames` incomplete. Missing or wrong-typed payload data is not treated as empty; its byte length is `null`, and a text payload shape is unknown. Numeric opcode `0` and an observed empty payload remain distinct from missing data. Opcode is nullable in the public inspection schema, so clients must handle unknown protocol values. When shape capture is selected, text messages are classified as text or complete value-free JSON shape; binary bytes, hashes, prefixes, and raw payloads are never retained. - Storage observations always redact values. Key names, IndexedDB names, and cache names require `include_storage_keys`; stable content fingerprints require the additional `include_storage_fingerprints` selection. Cookie fingerprints cover cookies applicable to the current main-frame URL, including its path, rather than every cookie path on that origin. Other storage remains scoped to the origin. Cache fingerprints hash all decoded response bytes, including bodies above 64 KiB, subject to the [64 MiB per-message CDP limit](#request-boundary). Missing or undecodable cache bodies and partial IndexedDB remote objects make fingerprint coverage incomplete, so identical observations remain `unknown`. - Unavailable or malformed storage inventories mark `storage_keys` unavailable, even without fingerprints. Comparisons suppress inferred key additions and removals while preserving modifications proven by complete fingerprints on both sides. Previously exported captures are not retroactively corrected; recapture them to obtain these coverage markers. - Script metadata is included only when CDP supplies a URL on an allowed origin. Stable keys exclude transient CDP script IDs, and exact transient raw URLs are used only during script/resource reconciliation. URL-less scripts are excluded because their origin cannot be established. When CDP supplies an execution-context association, the accepted script retains its authorized frame ID for later attribution. Script metadata admission bounds retained identity and source-map strings to 8 MiB and 50,000 records. When a source-map URL cannot fit, the script identity remains and source-map coverage reports `resource_budget_exhausted`; rejected script observations are counted without retaining their IDs. Source content is omitted unless explicitly requested and is returned inline as a self-verifying artifact with its SHA-256 digest. diff --git a/src/browser/CdpCaptureEventHandlers.ts b/src/browser/CdpCaptureEventHandlers.ts index 5d0792da55..b1b8b2ad67 100644 --- a/src/browser/CdpCaptureEventHandlers.ts +++ b/src/browser/CdpCaptureEventHandlers.ts @@ -509,28 +509,41 @@ export const handleWebSocketFrame = ( const connection = state.webSocketConnectionsById.get(requestId); if (connection === undefined) return; const response = recordValue(params.response); - const payload = cdpStringValue(response?.payloadData) ?? ""; - const opcode = Math.max(0, Math.trunc(numberValue(response?.opcode) ?? 0)); - const decodedBytes = - opcode === 1 ? undefined : paddedBase64ByteLength(payload); - if (opcode !== 1 && decodedBytes === undefined) + const opcodeValue = numberValue(response?.opcode); + // RFC 6455 encodes opcodes in four bits; keep reserved values as observed data. + const opcode = + opcodeValue !== undefined && + Number.isInteger(opcodeValue) && + opcodeValue >= 0 && + opcodeValue <= 15 + ? opcodeValue + : null; + const payload = cdpStringValue(response?.payloadData); + const payloadBytes = + opcode === null || payload === undefined + ? null + : opcode === 1 + ? Buffer.byteLength(payload) + : (paddedBase64ByteLength(payload) ?? null); + if (opcode === null || payloadBytes === null) state.completeness.exclude("websocket_frames", "invalid_protocol_value"); connection.events.push({ direction, opcode, - payload_bytes: - opcode === 1 ? Buffer.byteLength(payload) : (decodedBytes ?? null), + payload_bytes: payloadBytes, payload_shape: captureWebSocketShape(state, payload, opcode), }); }; const captureWebSocketShape = ( state: CdpCaptureEventsState, - payload: string, - opcode: number, + payload: string | undefined, + opcode: number | null, ): WebPageInspection["network"]["websocket_connections"][number]["events"][number]["payload_shape"] => { if (!state.input.include_websocket_shapes) return null; + if (opcode === null) return null; if (opcode !== 1) return { format: "binary", json_shape: null }; + if (payload === undefined) return null; const shape = inferJsonShape(payload); return { format: shape === null ? "text" : "json", diff --git a/src/browser/CdpCaptureEvents.webSocketFrames.test.ts b/src/browser/CdpCaptureEvents.webSocketFrames.test.ts new file mode 100644 index 0000000000..3c3fa08439 --- /dev/null +++ b/src/browser/CdpCaptureEvents.webSocketFrames.test.ts @@ -0,0 +1,182 @@ +import { expect, it } from "vitest"; + +import { inspectWebPageInputSchema } from "../domain/browserObservation.js"; +import { webPageInspectionSchema } from "../domain/browserObservationSchemas.js"; +import { CdpCaptureEvents } from "./CdpCaptureEvents.js"; + +const origin = "https://example.test"; + +const createEvents = (): CdpCaptureEvents => + new CdpCaptureEvents( + inspectWebPageInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: [origin], + target_id: "page", + include_websocket_shapes: true, + }), + new Set([origin]), + ); + +const captureFrame = ( + opcode: unknown, + payloadData: unknown, + includeOpcode = true, + includePayload = true, +) => { + const events = createEvents(); + events.ingest({ + method: "Network.webSocketCreated", + params: { requestId: "socket", url: "wss://example.test/live" }, + }); + events.ingest({ + method: "Network.webSocketFrameReceived", + params: { + requestId: "socket", + response: { + ...(includeOpcode ? { opcode } : {}), + ...(includePayload ? { payloadData } : {}), + }, + }, + }); + return { + events, + frame: events.webSocketConnectionsById.get("socket")?.events[0], + }; +}; + +it.each([ + { + label: "missing", + opcode: undefined, + payload: "", + hasOpcode: false, + hasPayload: true, + }, + { + label: "wrong type", + opcode: "1", + payload: "text", + hasOpcode: true, + hasPayload: true, + }, + { + label: "explicit null", + opcode: null, + payload: "", + hasOpcode: true, + hasPayload: true, + }, + { + label: "negative", + opcode: -1, + payload: "", + hasOpcode: true, + hasPayload: true, + }, + { + label: "fractional", + opcode: 1.5, + payload: "text", + hasOpcode: true, + hasPayload: true, + }, + { + label: "above opcode width", + opcode: 16, + payload: "", + hasOpcode: true, + hasPayload: true, + }, + { + label: "non-finite", + opcode: Number.NaN, + payload: "", + hasOpcode: true, + hasPayload: true, + }, +] as const)( + "keeps $label WebSocket opcode unknown", + ({ opcode, payload, hasOpcode, hasPayload }) => { + const { events, frame } = captureFrame( + opcode, + payload, + hasOpcode, + hasPayload, + ); + expect(frame).toEqual({ + direction: "received", + opcode: null, + payload_bytes: null, + payload_shape: null, + }); + expect(events.completeness.snapshot().excluded).toContainEqual({ + section: "websocket_frames", + reason: "invalid_protocol_value", + count: 1, + }); + expect( + webPageInspectionSchema.shape.network.shape.websocket_connections.element.safeParse( + { + request_id: "socket", + url: "wss://example.test/live", + events: [frame], + }, + ).success, + ).toBe(true); + }, +); + +it.each([ + { label: "continuation", opcode: 0, payload: "", bytes: 0, format: "binary" }, + { label: "empty text", opcode: 1, payload: "", bytes: 0, format: "text" }, + { + label: "reserved four-bit opcode", + opcode: 15, + payload: "", + bytes: 0, + format: "binary", + }, +] as const)( + "preserves observed $label data", + ({ opcode, payload, bytes, format }) => { + const { events, frame } = captureFrame(opcode, payload); + expect(frame).toEqual({ + direction: "received", + opcode, + payload_bytes: bytes, + payload_shape: { format, json_shape: null }, + }); + expect(events.completeness.snapshot().excluded).not.toContainEqual( + expect.objectContaining({ + section: "websocket_frames", + reason: "invalid_protocol_value", + }), + ); + }, +); + +it.each([ + { + label: "missing payload", + opcode: 1, + payload: undefined, + hasPayload: false, + }, + { label: "wrong-type payload", opcode: 1, payload: 42, hasPayload: true }, +] as const)( + "does not fabricate an empty payload for $label", + ({ opcode, payload, hasPayload }) => { + const { events, frame } = captureFrame(opcode, payload, true, hasPayload); + expect(frame).toEqual({ + direction: "received", + opcode, + payload_bytes: null, + payload_shape: null, + }); + expect(events.completeness.snapshot().excluded).toContainEqual({ + section: "websocket_frames", + reason: "invalid_protocol_value", + count: 1, + }); + }, +); diff --git a/src/domain/browserObservationSchemas.ts b/src/domain/browserObservationSchemas.ts index 53834085a4..5be597f175 100644 --- a/src/domain/browserObservationSchemas.ts +++ b/src/domain/browserObservationSchemas.ts @@ -185,7 +185,7 @@ const browserConsoleEventSchema = z.object({ }); const browserWebSocketFrameSchema = z.object({ direction: z.enum(["sent", "received"]), - opcode: z.number().int().min(0), + opcode: z.number().int().min(0).max(15).nullable(), payload_bytes: z.number().int().min(0).nullable(), payload_shape: z .object({ diff --git a/src/domain/webCaptureDiff.network.test.ts b/src/domain/webCaptureDiff.network.test.ts index cf1640a1c0..a736414f74 100644 --- a/src/domain/webCaptureDiff.network.test.ts +++ b/src/domain/webCaptureDiff.network.test.ts @@ -219,6 +219,74 @@ describe("passive network inventory changes", () => { }); }); +describe("unknown WebSocket opcode evidence", () => { + const captureWithWebSocketFrames = ( + events: WebPageInspection["network"]["websocket_connections"][number]["events"], + ): WebPageInspection => { + const capture = unselectedShapes(passiveNetworkCapture([])); + capture.network.websocket_connections.push({ + request_id: "capture-local-socket-id", + url: "wss://app.example.test/live", + events: structuredClone(events), + }); + return capture; + }; + + it("does not compare an unknown opcode as a real continuation frame", () => { + const malformed = captureWithWebSocketFrames([ + { + direction: "received", + opcode: null, + payload_bytes: null, + payload_shape: null, + }, + ]); + const observedZero = captureWithWebSocketFrames([ + { + direction: "received", + opcode: 0, + payload_bytes: 0, + payload_shape: { format: "binary", json_shape: null }, + }, + ]); + + for (const [before, after] of [ + [malformed, observedZero], + [observedZero, malformed], + ] as const) + expect(compare(before, after)).toMatchObject({ + status: "unknown", + total_changes: 0, + changes: [], + }); + }); + + it("still compares known payload facts in a stream with another unknown frame", () => { + const before = captureWithWebSocketFrames([ + { + direction: "received", + opcode: null, + payload_bytes: null, + payload_shape: null, + }, + { + direction: "sent", + opcode: 1, + payload_bytes: 3, + payload_shape: { format: "text", json_shape: null }, + }, + ]); + const after = structuredClone(before); + after.network.websocket_connections[0]!.events[1]!.payload_bytes = 4; + + expect(compare(before, after)).toMatchObject({ + status: "changed", + total_changes: 1, + changes: [{ change: "modified" }], + }); + }); +}); + describe("passive network coverage uncertainty", () => { it.each([ "attach_limited_sections", diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index ee0709ca1f..33a16f859e 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -367,6 +367,26 @@ const networkInventoryComplete = ( (section) => inspection.completeness.dropped_events[section] === 0, ); +type WebSocketEvent = + WebPageInspection["network"]["websocket_connections"][number]["events"][number]; + +const webSocketEventHasOpcode = ( + event: WebSocketEvent | undefined, +): event is WebSocketEvent => event !== undefined && event.opcode !== null; + +const webSocketEventHasPayloadBytes = ( + event: WebSocketEvent | undefined, +): boolean => webSocketEventHasOpcode(event) && event.payload_bytes !== null; + +const webSocketEventHasPayloadShape = ( + event: WebSocketEvent | undefined, +): boolean => webSocketEventHasOpcode(event) && event.payload_shape !== null; + +const webSocketOpcodesKnown = (inspection: WebPageInspection): boolean => + inspection.network.websocket_connections.every((connection) => + connection.events.every(webSocketEventHasOpcode), + ); + const networkDimension = ( before: WebPageInspection, after: WebPageInspection, @@ -374,11 +394,12 @@ const networkDimension = ( const requestsComplete = [before, after].every((inspection) => networkInventoryComplete(inspection, ["network_requests"]), ); - const webSocketInventoryComplete = [before, after].every((inspection) => - networkInventoryComplete(inspection, [ - "websocket_connections", - "websocket_frames", - ]), + const webSocketInventoryComplete = [before, after].every( + (inspection) => + networkInventoryComplete(inspection, [ + "websocket_connections", + "websocket_frames", + ]) && webSocketOpcodesKnown(inspection), ); const requestIdentities = networkIdentityCoverage(before, after); const metadata = sharedNetworkMetadata(before, after); @@ -432,12 +453,12 @@ const networkDimension = ( (inspection) => sectionsComplete(inspection, ["websocket_shapes"]) && inspection.network.websocket_connections.every((connection) => - connection.events.every((event) => event.payload_shape !== null), + connection.events.every(webSocketEventHasPayloadShape), ), ); const webSocketPayloadBytesComplete = [before, after].every((inspection) => inspection.network.websocket_connections.every((connection) => - connection.events.every((event) => event.payload_bytes !== null), + connection.events.every(webSocketEventHasPayloadBytes), ), ); const beforeWebSockets = webSocketGroups(before); @@ -633,11 +654,11 @@ const webSocketPayloadMaps = ( const comparableShapes = new Set(); for (let index = 0; index < eventCount; index += 1) { if ( - beforeGroup.connections.every( - (connection) => connection.events[index]?.payload_bytes !== null, + beforeGroup.connections.every((connection) => + webSocketEventHasPayloadBytes(connection.events[index]), ) && - afterGroup.connections.every( - (connection) => connection.events[index]?.payload_bytes !== null, + afterGroup.connections.every((connection) => + webSocketEventHasPayloadBytes(connection.events[index]), ) ) comparableBytes.add(index); @@ -645,11 +666,11 @@ const webSocketPayloadMaps = ( // on both sides; this avoids assigning an unknown shape to a known one. if ( selected && - beforeGroup.connections.every( - (connection) => connection.events[index]?.payload_shape !== null, + beforeGroup.connections.every((connection) => + webSocketEventHasPayloadShape(connection.events[index]), ) && - afterGroup.connections.every( - (connection) => connection.events[index]?.payload_shape !== null, + afterGroup.connections.every((connection) => + webSocketEventHasPayloadShape(connection.events[index]), ) ) comparableShapes.add(index); diff --git a/tests/acceptance/browser/webSocketOpcodeCliMcp.test.ts b/tests/acceptance/browser/webSocketOpcodeCliMcp.test.ts new file mode 100644 index 0000000000..e4438f55eb --- /dev/null +++ b/tests/acceptance/browser/webSocketOpcodeCliMcp.test.ts @@ -0,0 +1,148 @@ +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const websocketEvents = (origin: string) => { + const cases = [ + { requestId: "missing-opcode", response: { payloadData: "text" } }, + { + requestId: "wrong-type-opcode", + response: { opcode: "1", payloadData: "text" }, + }, + { requestId: "null-opcode", response: { opcode: null, payloadData: "" } }, + { requestId: "negative-opcode", response: { opcode: -1, payloadData: "" } }, + { + requestId: "fractional-opcode", + response: { opcode: 1.5, payloadData: "text" }, + }, + { requestId: "observed-zero", response: { opcode: 0, payloadData: "" } }, + { requestId: "observed-text", response: { opcode: 1, payloadData: "" } }, + { requestId: "bad-width", response: { opcode: 16, payloadData: "" } }, + { requestId: "missing-payload", response: { opcode: 1 } }, + ]; + return cases.flatMap(({ requestId, response }) => [ + { + method: "Network.webSocketCreated", + params: { requestId, url: `${origin.replace(/^http/u, "ws")}/live` }, + }, + { + method: "Network.webSocketFrameReceived", + params: { requestId, response }, + }, + ]); +}; + +cliTest( + "preserves unknown WebSocket opcodes and known zero through public CLI and stdio MCP", + async ({ cli }) => { + const { call } = await connectLocalToolsMcp(); + const browser = await startFakeCdpBrowser({ + eventsBeforeReply: ["Network.enable"], + commandEvents: (command, origin) => + command.method === "Network.enable" + ? websocketEvents(origin).map((event) => ({ + ...event, + ...(command.sessionId === undefined + ? {} + : { sessionId: command.sessionId }), + })) + : undefined, + }); + try { + const cliResponse = await cli.run({ + arguments: [ + "inspect-web-page", + browser.endpoint, + "allowed-page", + "--allowed-origins", + browser.allowedOrigin, + "--observation-ms", + "0", + "--include-websocket-shapes", + "--json", + ], + }); + expect(cliResponse.exitCode, cliResponse.stdout).toBe(0); + const cliContract = toolContract("inspect_web_page").outputSchema.parse( + cliResponse.json, + ); + const cliEvidence = parseEvidence(cliContract); + + const mcpResponse = await call("inspect_web_page", { + cdp_endpoint: browser.endpoint, + target_id: "allowed-page", + allowed_origins: [browser.allowedOrigin], + observation_ms: 0, + include_websocket_shapes: true, + }); + expect(mcpResponse.isError).not.toBe(true); + const mcpEvidence = parseEvidence( + toolContract("inspect_web_page").outputSchema.parse( + mcpResponse.structuredContent, + ), + ); + + for (const evidence of [cliEvidence, mcpEvidence]) { + const page = webPageInspectionSchema.parse(evidence.normalized_result); + const frames = new Map( + page.network.websocket_connections.map((connection) => [ + connection.request_id, + connection.events[0], + ]), + ); + expect(frames.get("missing-opcode")).toMatchObject({ + opcode: null, + payload_bytes: null, + payload_shape: null, + }); + expect(frames.get("wrong-type-opcode")).toMatchObject({ + opcode: null, + payload_bytes: null, + payload_shape: null, + }); + expect(frames.get("bad-width")).toMatchObject({ + opcode: null, + payload_bytes: null, + payload_shape: null, + }); + for (const requestId of [ + "null-opcode", + "negative-opcode", + "fractional-opcode", + ]) + expect(frames.get(requestId)).toMatchObject({ + opcode: null, + payload_bytes: null, + payload_shape: null, + }); + expect(frames.get("missing-payload")).toMatchObject({ + opcode: 1, + payload_bytes: null, + payload_shape: null, + }); + expect(frames.get("observed-zero")).toMatchObject({ + opcode: 0, + payload_bytes: 0, + payload_shape: { format: "binary", json_shape: null }, + }); + expect(frames.get("observed-text")).toMatchObject({ + opcode: 1, + payload_bytes: 0, + payload_shape: { format: "text", json_shape: null }, + }); + expect(page.completeness.excluded).toContainEqual({ + section: "websocket_frames", + reason: "invalid_protocol_value", + count: 7, + }); + } + } finally { + await browser.close(); + } + }, +); From 4108d1363a823ee39f199de64be860cf6b0162d5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:27:56 +0800 Subject: [PATCH 15/35] test(cli): align Kuna readiness and cancellation expectations --- tests/boundary/cli/doctorCancellation.test.ts | 22 +++++++++++++------ .../cli/setupProviderReadiness.test.ts | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/boundary/cli/doctorCancellation.test.ts b/tests/boundary/cli/doctorCancellation.test.ts index 7904d0aa5c..530427dcc6 100644 --- a/tests/boundary/cli/doctorCancellation.test.ts +++ b/tests/boundary/cli/doctorCancellation.test.ts @@ -40,11 +40,13 @@ cliTest.skipIf(!["darwin", "linux"].includes(process.platform)).for( const executable = provider === "kuna" ? join(root, "kuna") : join(jdk, "bin", "java"); const probeArgument = provider === "kuna" ? "--version" : "--list-modules"; + const runIdField = + provider === "jadx" ? ",runId:process.env.REA_PROCESS_RUN_ID" : ""; await writeFile( executable, `#!${process.execPath} if (!process.argv.slice(2).includes(${JSON.stringify(probeArgument)})) process.exit(7); -require("node:fs").writeFileSync(${JSON.stringify(ready)}, JSON.stringify({pid: process.pid, runId: process.env.REA_PROCESS_RUN_ID})); +require("node:fs").writeFileSync(${JSON.stringify(ready)}, JSON.stringify({pid: process.pid${runIdField}})); process.on("SIGTERM", () => {}); setInterval(() => { if (require("node:fs").existsSync(${JSON.stringify(release)})) process.exit(0); }, 25).unref(); setTimeout(() => process.exit(99), 20000); @@ -94,12 +96,18 @@ setTimeout(() => process.exit(99), 20000); { timeout: 10_000 }, ) .toBe(true); - const observed = z - .strictObject({ - pid: z.number().int().positive(), - runId: z.string().uuid().optional(), - }) - .parse(JSON.parse(await readFile(ready, "utf8"))); + const readyRecord: unknown = JSON.parse(await readFile(ready, "utf8")); + const observed: { readonly pid: number; readonly runId?: string } = + provider === "jadx" + ? z + .strictObject({ + pid: z.number().int().positive(), + runId: z.string().uuid(), + }) + .parse(readyRecord) + : z + .strictObject({ pid: z.number().int().positive() }) + .parse(readyRecord); if (provider === "jadx") { if (observed.runId === undefined) throw new Error("Missing JADX run token"); diff --git a/tests/boundary/cli/setupProviderReadiness.test.ts b/tests/boundary/cli/setupProviderReadiness.test.ts index 88f215735c..786df8ff66 100644 --- a/tests/boundary/cli/setupProviderReadiness.test.ts +++ b/tests/boundary/cli/setupProviderReadiness.test.ts @@ -8,7 +8,7 @@ import { cliTest } from "../../support/cli/cliFixture.js"; import { writeFakeKuna } from "../providers/kuna/fakeKuna.js"; cliTest.skipIf(process.platform === "win32").for(["blocked", "ready"] as const)( - "setup dry-run advertises Kuna only with a %s cache", + "setup dry-run reports Kuna readiness for a %s cache", async (cache, { cli }) => { const root = await createTestTempDirectory("rea-setup-provider-readiness-"); const home = join(root, "home"); @@ -37,7 +37,7 @@ cliTest.skipIf(process.platform === "win32").for(["blocked", "ready"] as const)( providerInspections: expect.arrayContaining([ expect.objectContaining({ id: "kuna", - available: true, + available: cache === "ready", checks: expect.arrayContaining([ expect.objectContaining({ name: "cache", From 5518363aa625db75590a163d3174d767d97064ba Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:28:03 +0800 Subject: [PATCH 16/35] test(artifacts): compare matching resource-root workflows --- scripts/lib/artifact-e2e.mjs | 17 +++++++++++--- scripts/verify/apple/asset-catalog.mjs | 17 +++++++++++++- scripts/verify/apple/interface-builder.mjs | 22 +++++++++++++++++-- src/cliCommandNames.ts | 2 ++ .../interfaceBuilderHierarchyLimits.test.ts | 4 ++-- .../applications/nibFieldIdentity.test.ts | 2 +- .../artifactProvider.directAnalysis.test.ts | 4 +++- tests/support/mcp/artifactSession.ts | 22 ++++++++++++++++--- 8 files changed, 77 insertions(+), 13 deletions(-) diff --git a/scripts/lib/artifact-e2e.mjs b/scripts/lib/artifact-e2e.mjs index b0ee1c2c13..444639319e 100644 --- a/scripts/lib/artifact-e2e.mjs +++ b/scripts/lib/artifact-e2e.mjs @@ -58,8 +58,7 @@ export async function artifactCliEvidence( return evidence; } -/** Run a real stdio MCP subprocess, owning and closing its transport. */ -export async function withArtifactMcp(target, verify) { +const withOpenedArtifactMcp = async (target, openTool, verify) => { const transport = new StdioClientTransport({ command: process.execPath, args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"], @@ -70,15 +69,27 @@ export async function withArtifactMcp(target, verify) { try { await client.connect(transport); const opened = await client.callTool({ - name: "open_binary", + name: openTool, arguments: { path: target }, }); assert.notEqual(opened.isError, true, JSON.stringify(opened)); + if (openTool === "open_bundle_resources") + assert.equal(opened.structuredContent?.result?.caller_path, target); await verify(client); } finally { await client.close(); await transport.close(); } +}; + +/** Run a real stdio MCP subprocess with an explicitly opened binary target. */ +export async function withArtifactMcp(target, verify) { + return withOpenedArtifactMcp(target, "open_binary", verify); +} + +/** Run a real stdio MCP subprocess with an explicitly opened bundle resource root. */ +export async function withBundleResourcesMcp(target, verify) { + return withOpenedArtifactMcp(target, "open_bundle_resources", verify); } /** Require a successful structured MCP tool result rather than its text rendering. */ diff --git a/scripts/verify/apple/asset-catalog.mjs b/scripts/verify/apple/asset-catalog.mjs index f29d79386a..4de295f06f 100644 --- a/scripts/verify/apple/asset-catalog.mjs +++ b/scripts/verify/apple/asset-catalog.mjs @@ -10,6 +10,7 @@ import { artifactCli, artifactMcpResult, withArtifactMcp, + withBundleResourcesMcp, } from "../../lib/artifact-e2e.mjs"; if (process.platform !== "darwin") @@ -68,6 +69,9 @@ try { const sha256 = createHash("sha256") .update(await readFile(join(resources, "Assets.car"))) .digest("hex"); + const executableSha256 = createHash("sha256") + .update(await readFile(join(app, "Contents", "MacOS", "Fixture"))) + .digest("hex"); const cli = await artifactCli("inspect-asset-catalog", app); assert.deepEqual( cli.records.map(({ metadata }) => metadata), @@ -80,7 +84,7 @@ try { asset_name === "FixtureColor" && metadata.AssetType === "Color", ), ); - await withArtifactMcp(app, async (client) => { + await withBundleResourcesMcp(app, async (client) => { assert.deepEqual( await artifactMcpResult(client, "inspect_asset_catalog"), cli, @@ -99,6 +103,17 @@ try { }); assert.equal(invalid.isError, true); }); + const { resource_root, target_sha256, ...resourcePayload } = cli; + assert.equal(resource_root.executable_sha256, null); + assert.equal(target_sha256, null); + await withArtifactMcp(app, async (client) => { + const binary = await artifactMcpResult(client, "inspect_asset_catalog"); + assert.equal(binary.target_sha256, executableSha256); + assert.deepEqual(binary, { + ...resourcePayload, + target_sha256: executableSha256, + }); + }); report = { ok: true, mocked: false, diff --git a/scripts/verify/apple/interface-builder.mjs b/scripts/verify/apple/interface-builder.mjs index a27bf656b7..1d9d3b5d99 100644 --- a/scripts/verify/apple/interface-builder.mjs +++ b/scripts/verify/apple/interface-builder.mjs @@ -1,8 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; -import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -12,6 +13,7 @@ import { artifactCliEvidence, artifactMcpResult, withArtifactMcp, + withBundleResourcesMcp, } from "../../lib/artifact-e2e.mjs"; const exec = promisify(execFile); @@ -49,7 +51,10 @@ try { appPath, ); const decoded = evidence.normalized_result; - await withArtifactMcp(appPath, async (client) => { + const executableSha256 = createHash("sha256") + .update(await readFile(executablePath)) + .digest("hex"); + await withBundleResourcesMcp(appPath, async (client) => { const mcp = await artifactMcpResult(client, "decode_interface_builder"); assert.deepEqual( mcp, @@ -57,6 +62,19 @@ try { "CLI and MCP compiled archive results differ", ); }); + const { resource_root, target_sha256, ...resourcePayload } = decoded; + assert.equal(resource_root.executable_sha256, null); + assert.equal(target_sha256, null); + await withArtifactMcp(appPath, async (client) => { + const binary = await artifactMcpResult(client, "decode_interface_builder"); + assert.equal(binary.target_sha256, executableSha256); + assert.equal(binary.graph.target_sha256, executableSha256); + assert.deepEqual(binary, { + ...resourcePayload, + target_sha256: executableSha256, + graph: { ...decoded.graph, target_sha256: executableSha256 }, + }); + }); assert.equal(decoded.documents.length, 1); assert.equal( decoded.documents[0]?.relative_path, diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index ec23e6508a..ce6c6bf837 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -193,6 +193,8 @@ export const CLI_COMMAND_TOOL_ALIASES = Object.freeze({ /** MCP operations without a dedicated CLI command or direct command alias. */ export const MCP_TOOLS_WITHOUT_DEDICATED_CLI = Object.freeze([ + // MCP session control selects a resource root; CLI resource commands take paths directly. + "open_bundle_resources", "address_name", "comment", "current_address", diff --git a/tests/acceptance/applications/interfaceBuilderHierarchyLimits.test.ts b/tests/acceptance/applications/interfaceBuilderHierarchyLimits.test.ts index af508592cf..3bcd6ce9a9 100644 --- a/tests/acceptance/applications/interfaceBuilderHierarchyLimits.test.ts +++ b/tests/acceptance/applications/interfaceBuilderHierarchyLimits.test.ts @@ -10,7 +10,7 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; import { createTestMcp } from "../../support/mcp/mcpFixture.js"; -import { openArtifactTarget } from "../../support/mcp/artifactSession.js"; +import { openBundleResourcesTarget } from "../../support/mcp/artifactSession.js"; const verifyMcpParity = async ( app: string, @@ -18,7 +18,7 @@ const verifyMcpParity = async ( ): Promise => { const mcp = createTestMcp(); onTestFinished(async () => mcp.close()); - const client = await openArtifactTarget(mcp, onTestFinished, app); + const client = await openBundleResourcesTarget(mcp, onTestFinished, app); const complete = await client.callTool({ name: "decode_interface_builder", arguments: { max_objects: 2 }, diff --git a/tests/acceptance/applications/nibFieldIdentity.test.ts b/tests/acceptance/applications/nibFieldIdentity.test.ts index 5af16f890f..82aa769d19 100644 --- a/tests/acceptance/applications/nibFieldIdentity.test.ts +++ b/tests/acceptance/applications/nibFieldIdentity.test.ts @@ -66,7 +66,7 @@ cliTest( try { await client.connect(transport); const opened = await client.callTool({ - name: "open_binary", + name: "open_bundle_resources", arguments: { path: app }, }); expect(opened.isError, JSON.stringify(opened)).not.toBe(true); diff --git a/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts b/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts index f6fb12853b..1cb7b9239b 100644 --- a/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts +++ b/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts @@ -48,7 +48,9 @@ it("runs static .app decoding without registering native analysis engines", asyn evidence.normalized_result, ); expect(result.documents).toEqual([]); - expect(result.graph.target_sha256).toBe(evidence.subject?.digest.sha256); + expect(result.graph.target_sha256).toBe( + evidence.subject?.digest.sha256 ?? null, + ); }); it("inspects an artifact through the direct CLI workflow without an REA grant", async () => { diff --git a/tests/support/mcp/artifactSession.ts b/tests/support/mcp/artifactSession.ts index 569724b9b6..53ca015602 100644 --- a/tests/support/mcp/artifactSession.ts +++ b/tests/support/mcp/artifactSession.ts @@ -5,10 +5,10 @@ import { createArtifactBinarySession } from "../../../src/composition/binary.js" import { createServer } from "../../../src/server/createServer.js"; import type { TestMcp } from "./mcpFixture.js"; -/** Open a static artifact target in the production artifact-only MCP composition. */ -export const openArtifactTarget = async ( +const openSessionTarget = async ( mcp: TestMcp, onTestFinished: (handler: OnTestFinishedHandler) => void, + operation: "open_binary" | "open_bundle_resources", path: string, ): Promise => { const session = await createArtifactBinarySession(process.env); @@ -17,9 +17,25 @@ export const openArtifactTarget = async ( }); const client = await mcp.connect(createServer({ kind: "session", session })); const opened = await client.callTool({ - name: "open_binary", + name: operation, arguments: { path }, }); expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); return client; }; + +/** Open a static artifact target in the production artifact-only MCP composition. */ +export const openArtifactTarget = ( + mcp: TestMcp, + onTestFinished: (handler: OnTestFinishedHandler) => void, + path: string, +): Promise => + openSessionTarget(mcp, onTestFinished, "open_binary", path); + +/** Open a caller-selected bundle resource root in the production MCP session. */ +export const openBundleResourcesTarget = ( + mcp: TestMcp, + onTestFinished: (handler: OnTestFinishedHandler) => void, + path: string, +): Promise => + openSessionTarget(mcp, onTestFinished, "open_bundle_resources", path); From 60956c8c56475e7aaab9559f780908851677ced6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:28:08 +0800 Subject: [PATCH 17/35] fix(ci): run cache retention from its trusted workflow revision --- .github/workflows/cache-retention.yml | 25 +- .../boundary/filesystem/ciCachePolicy.test.ts | 249 ++++++++++++++++-- 2 files changed, 249 insertions(+), 25 deletions(-) diff --git a/.github/workflows/cache-retention.yml b/.github/workflows/cache-retention.yml index 70545e3ebe..580f2d1fa3 100644 --- a/.github/workflows/cache-retention.yml +++ b/.github/workflows/cache-retention.yml @@ -18,10 +18,11 @@ concurrency: jobs: retain: name: Retain Turbo snapshots and npm download caches - # Checkout and run only a same-repository default-branch commit. Pull - # requests, including forks, do not reach these steps. Static and build - # save Turbo snapshots before later lanes finish, so a failed, timed-out, - # or cancelled run still retains those snapshots. + # Restrict the triggering-source checkout to same-repository default-branch + # CI events. The helper runs from this trusted workflow revision; triggering + # source is used only for cache identity and is never executed. Static and + # build save snapshots before later lanes finish, so failed, timed-out, or + # cancelled runs still retain those snapshots. if: >- (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure' || @@ -35,23 +36,29 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.workflow_sha }} + path: trusted-retention + persist-credentials: false - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ github.event.workflow_run.head_sha }} + path: triggering-source persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: - node-version-file: .nvmrc + node-version-file: trusted-retention/.nvmrc - name: Retain current Turbo snapshots and npm download caches env: GH_TOKEN: ${{ github.token }} CI_REPOSITORY: ${{ github.repository }} CACHE_PREFIX: rea-static-v1-${{ runner.os }}-${{ runner.arch }}- - CACHE_KEEP_KEY: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }} + CACHE_KEEP_KEY: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }} CACHE_DEEP_PREFIX: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}- - CACHE_DEEP_KEEP_KEY: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }} + CACHE_DEEP_KEEP_KEY: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }} CACHE_KNIP_PREFIX: knip-meta-v1-${{ runner.os }}- - CACHE_KNIP_KEEP_KEY: knip-meta-v1-${{ runner.os }}-${{ hashFiles('knip.json', 'tsconfig.json', 'package.json', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }} + CACHE_KNIP_KEEP_KEY: knip-meta-v1-${{ runner.os }}-${{ hashFiles('triggering-source/knip.json', 'triggering-source/tsconfig.json', 'triggering-source/package.json', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }} CACHE_PORTABLE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}- DEDUPE_NPM_DOWNLOADS: "true" - run: node scripts/ci/cache-retention.mjs + run: node trusted-retention/scripts/ci/cache-retention.mjs diff --git a/tests/boundary/filesystem/ciCachePolicy.test.ts b/tests/boundary/filesystem/ciCachePolicy.test.ts index 2b70aa82c2..6b5b22f913 100644 --- a/tests/boundary/filesystem/ciCachePolicy.test.ts +++ b/tests/boundary/filesystem/ciCachePolicy.test.ts @@ -1,5 +1,14 @@ import { execFileSync } from "node:child_process"; -import { chmod, readFile, readdir, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { + access, + chmod, + copyFile, + mkdir, + readFile, + readdir, + writeFile, +} from "node:fs/promises"; import { delimiter, join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -61,6 +70,185 @@ const readWorkflows = async () => { ); }; +function aggregateHashFilesContent(contents: Buffer[]) { + const aggregate = createHash("sha256"); + for (const content of contents) + aggregate.update(createHash("sha256").update(content).digest()); + return aggregate.digest("hex"); +} + +async function verifyPrefixedHashFiles( + retentionEnv: Record, + runnerWorkspace: string, + legacyRoot: string, +) { + const keyExpressions = [ + retentionEnv.CACHE_KEEP_KEY, + retentionEnv.CACHE_DEEP_KEEP_KEY, + retentionEnv.CACHE_KNIP_KEEP_KEY, + ]; + let targetHash: string | undefined; + for (const expression of keyExpressions) { + if (expression === undefined) + throw new Error("Missing retention cache key"); + const paths = expression.match(/hashFiles\(([^)]*)\)/u)?.[1]; + expect(paths).toBeDefined(); + const targetPaths = paths!.split(", ").map((path) => path.slice(1, -1)); + expect( + targetPaths.every((path) => path.startsWith("triggering-source/")), + ).toBe(true); + const fileNames = targetPaths.map((path) => + path.slice("triggering-source/".length), + ); + const nestedHash = aggregateHashFilesContent( + await Promise.all( + targetPaths.map((path) => readFile(join(runnerWorkspace, path))), + ), + ); + const flatHash = aggregateHashFilesContent( + await Promise.all( + fileNames.map((name) => readFile(join(legacyRoot, name))), + ), + ); + expect(nestedHash).toBe(flatHash); + if (expression === retentionEnv.CACHE_KEEP_KEY) targetHash = nestedHash; + } + expect(targetHash).toBeDefined(); + return targetHash!; +} + +async function exerciseTrustedRetentionWithLegacyTarget( + retentionRun: unknown, + retentionEnv: Record, +) { + const runnerWorkspace = await createTestTempDirectory( + "rea-ci-retention-trusted-checkout-", + ); + const legacyTarget = join(runnerWorkspace, "triggering-source"); + const legacyRoot = join(runnerWorkspace, "legacy-root"); + const trustedSource = join(runnerWorkspace, "trusted-retention"); + const inputNames = [ + ".nvmrc", + ".npmrc", + "package-lock.json", + "knip.json", + "tsconfig.json", + "package.json", + ]; + const targetSha = "0123456789abcdef0123456789abcdef01234567"; + await mkdir(legacyTarget, { recursive: true }); + await mkdir(legacyRoot, { recursive: true }); + for (const name of inputNames) { + const contents = `target-source:${name}\n`; + await writeFile(join(legacyTarget, name), contents); + await writeFile(join(legacyRoot, name), contents); + } + const trustedHelperPath = join( + trustedSource, + "scripts/ci/cache-retention.mjs", + ); + await mkdir(join(trustedSource, "scripts/ci"), { recursive: true }); + await copyFile( + join(repository, "scripts/ci/cache-retention.mjs"), + trustedHelperPath, + ); + await expect( + access(join(legacyTarget, "scripts/ci/cache-retention.mjs")), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect(access(trustedHelperPath)).resolves.toBeUndefined(); + + // Actions hashFiles hashes file bytes, then the ordered per-file digests. + // A common checkout-directory prefix preserves both file contents and order. + const targetHash = await verifyPrefixedHashFiles( + retentionEnv, + runnerWorkspace, + legacyRoot, + ); + + const prefix = "rea-static-v1-Linux-X64-"; + const keepKey = `${prefix}${targetHash}-${targetSha}`; + const cacheFixture = { + actions_caches: [ + { + id: 1, + key: `${prefix}old`, + created_at: "2026-10-01T00:00:00Z", + last_accessed_at: "2026-10-01T00:00:00Z", + size_in_bytes: 10, + version: "a".repeat(64), + ref: "refs/heads/main", + }, + { + id: 2, + key: `${prefix}mid`, + created_at: "2026-10-15T00:00:00Z", + last_accessed_at: "2026-10-15T00:00:00Z", + size_in_bytes: 20, + version: "a".repeat(64), + ref: "refs/heads/main", + }, + { + id: 3, + key: `${prefix}new`, + created_at: "2026-10-20T00:00:00Z", + last_accessed_at: "2026-10-20T00:00:00Z", + size_in_bytes: 30, + version: "a".repeat(64), + ref: "refs/heads/main", + }, + { + id: 4, + key: keepKey, + created_at: "2026-09-30T00:00:00Z", + last_accessed_at: "2026-09-30T00:00:00Z", + size_in_bytes: 40, + version: "a".repeat(64), + ref: "refs/heads/main", + }, + ], + }; + const ghPath = join(runnerWorkspace, "bin/gh"); + const ghLog = join(runnerWorkspace, "fake-gh.log"); + const cacheInput = join(runnerWorkspace, "cache-list.json"); + await mkdir(join(runnerWorkspace, "bin"), { recursive: true }); + await writeFile(cacheInput, JSON.stringify(cacheFixture)); + await writeFile( + ghPath, + `#!${process.execPath}\nconst fs = require("node:fs");\nconst [command, option, method, endpoint] = process.argv.slice(2);\nif (command !== "api" || option !== "--method") process.exit(2);\nfs.appendFileSync(process.env.FAKE_GH_LOG, method + " " + endpoint + "\\n");\nif (method === "GET") process.stdout.write(fs.readFileSync(process.env.FAKE_GH_CACHE_INPUT, "utf8"));\nelse if (method !== "DELETE") process.exit(2);\n`, + ); + await chmod(ghPath, 0o755); + expect(retentionRun).toBe( + "node trusted-retention/scripts/ci/cache-retention.mjs", + ); + const helperPath = join( + runnerWorkspace, + String(retentionRun).replace(/^node /u, ""), + ); + const command = execFileSync(process.execPath, [helperPath], { + cwd: runnerWorkspace, + env: { + PATH: join(runnerWorkspace, "bin"), + CI_REPOSITORY: "morluto/rea", + CACHE_PREFIX: prefix, + CACHE_KEEP_KEY: keepKey, + FAKE_GH_LOG: ghLog, + FAKE_GH_CACHE_INPUT: cacheInput, + }, + encoding: "utf8", + }); + const requests = await readFile(ghLog, "utf8"); + expect(JSON.parse(command)).toEqual({ + dryRun: false, + families: [{ prefix, deleteCount: 1, deleteBytes: 10 }], + npmDuplicateDeleteCount: 0, + npmDuplicateDeleteBytes: 0, + }); + expect(requests.trim().split("\n")).toEqual([ + "GET repos/morluto/rea/actions/caches?per_page=100&page=1&key=rea-static-v1-Linux-X64-", + "DELETE repos/morluto/rea/actions/caches/1", + ]); +} + describe("hosted npm download-cache policy", () => { it("only allows shared writes for trusted original default-branch events", async () => { const action = await readAction(); @@ -345,29 +533,58 @@ it("deletes superseded caches only from the trusted follow-up workflow", async ( expect(job?.if).not.toContain("pull_request"); const steps = job?.steps ?? []; expect(steps.some((step) => step.run === "npm ci")).toBe(false); - const checkout = steps[0]; - expect(checkout?.uses).toBe( - "actions/checkout@11d5960a326750d5838078e36cf38b85af677262", + const checkouts = steps.filter( + (step) => + step.uses === "actions/checkout@11d5960a326750d5838078e36cf38b85af677262", ); - expect(checkout?.with).toMatchObject({ - ref: "${{ github.event.workflow_run.head_sha }}", + const trustedCheckout = checkouts[0]; + expect(trustedCheckout?.with).toMatchObject({ + ref: "${{ github.workflow_sha }}", + path: "trusted-retention", "persist-credentials": false, }); - const retention = steps.find( - (step) => step.run === "node scripts/ci/cache-retention.mjs", + const targetCheckout = checkouts[1]; + expect(targetCheckout?.with).toMatchObject({ + ref: "${{ github.event.workflow_run.head_sha }}", + path: "triggering-source", + "persist-credentials": false, + }); + expect(checkouts).toHaveLength(2); + expect(steps.some((step) => step.run === "npm ci")).toBe(false); + const setupNode = steps.find( + (step) => + step.uses === + "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020", ); + expect(setupNode?.with).toMatchObject({ + "node-version-file": "trusted-retention/.nvmrc", + }); + const retention = steps.find( + (step) => + step.run === "node trusted-retention/scripts/ci/cache-retention.mjs", + ); + expect(retention).toBeDefined(); const retentionEnv = z.record(z.string(), z.string()).parse(retention?.env); expect(retentionEnv.CACHE_PREFIX).toBe( "rea-static-v1-${{ runner.os }}-${{ runner.arch }}-", ); expect(retentionEnv.CACHE_KEEP_KEY).toBe( - "rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}", + "rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }}", + ); + expect(retentionEnv.CACHE_DEEP_KEEP_KEY).toContain( + "hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json')", + ); + expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toContain( + "hashFiles('triggering-source/knip.json', 'triggering-source/tsconfig.json', 'triggering-source/package.json', 'triggering-source/package-lock.json')", ); expect(retentionEnv.CACHE_PORTABLE_PREFIX).toBe( "rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-", ); expect(retentionEnv.DEDUPE_NPM_DOWNLOADS).toBe("true"); expect(retentionEnv.GH_TOKEN).toBe("${{ github.token }}"); + + // Exercise the selected trusted helper against a legacy target tree. + await exerciseTrustedRetentionWithLegacyTarget(retention?.run, retentionEnv); const ci = (await readWorkflows()).find(({ file }) => file === "ci.yml"); expect(JSON.stringify(ci?.workflow)).not.toContain('"actions":"write"'); const deepSteps = ci?.workflow.jobs?.["static-deep"]?.steps ?? []; @@ -382,18 +599,18 @@ it("deletes superseded caches only from the trusted follow-up workflow", async ( const savedOnThisRun = "${{ github.event.workflow_run.head_sha }}"; const deepPrefix = "rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-"; const knipPrefix = "knip-meta-v1-${{ runner.os }}-"; + const targetSourceKey = (id: string) => + publishedKey(id) + .replaceAll("${{ github.sha }}", savedOnThisRun) + .replaceAll("hashFiles('", "hashFiles('triggering-source/") + .replaceAll(", '", ", 'triggering-source/"); expect(retentionEnv.CACHE_DEEP_PREFIX).toBe(deepPrefix); expect(retentionEnv.CACHE_DEEP_KEEP_KEY).toBe( - publishedKey("static-cache").replaceAll( - "${{ github.sha }}", - savedOnThisRun, - ), + targetSourceKey("static-cache"), ); expect(publishedKey("static-cache").startsWith(deepPrefix)).toBe(true); expect(retentionEnv.CACHE_KNIP_PREFIX).toBe(knipPrefix); - expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toBe( - publishedKey("knip-cache").replaceAll("${{ github.sha }}", savedOnThisRun), - ); + expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toBe(targetSourceKey("knip-cache")); expect(publishedKey("knip-cache").startsWith(knipPrefix)).toBe(true); }); From ab58edcdf50b4bd687900ee2dc362ecc5c3dce15 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:33:35 +0800 Subject: [PATCH 18/35] fix(environment): retain tarball descriptor ownership during export Use the shared positional reader for both tarball passes so early output refusal cannot close the descriptor ahead of its owner. Closes #5684 --- .../environmentBundleWrite.test.ts | 53 +++++++++++++++++++ .../environment/environmentBundleWrite.ts | 8 ++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/application/environment/environmentBundleWrite.test.ts b/src/application/environment/environmentBundleWrite.test.ts index 4169cb2c44..6bbac53e89 100644 --- a/src/application/environment/environmentBundleWrite.test.ts +++ b/src/application/environment/environmentBundleWrite.test.ts @@ -1,4 +1,5 @@ import { + appendFile, chmod, mkdir, readFile, @@ -70,6 +71,58 @@ it("retains the admitted tarball descriptor and refuses a pathname replacement d } }); +it("keeps tarball ownership when the admitted inode grows during output", async () => { + const root = await createTestTempDirectory("rea-tarball-growth-"); + const files = await writeEnvironmentServeFiles(root); + const inputs = await readEnvironmentExportInputs( + files.manifestPath, + files.capturePath, + ); + if (!inputs.ok) throw inputs.error; + const tarball = join(root, "package.tgz"); + await writeFile(tarball, Buffer.alloc(2 * 1024 * 1024, 0x41)); + const directory = join(root, "bundle"); + const original = SafeOutputTree.prototype.write; + let grown = false; + const writer = vi + .spyOn(SafeOutputTree.prototype, "write") + .mockImplementation(async function ( + this: SafeOutputTree, + ...args: Parameters + ) { + const [relative] = args; + if (!grown && relative === "envs/grafana-example/rea-agents.tgz") { + grown = true; + await appendFile(tarball, Buffer.alloc(1024 * 1024, 0x42)); + } + return original.apply(this, args); + }); + try { + const result = await exportEnvironmentBundle({ + ...inputs.value, + source: environmentServeSource(), + directory, + packageTarball: tarball, + }); + expect(grown).toBe(true); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisOutputError", + operation: "environment-export", + reason: + "Extracted content exceeds the inventoried size: envs/grafana-example/rea-agents.tgz", + }, + }); + if (result.ok) + throw new Error("Expected the changed tarball to be refused"); + expect(result.error.cleanup).toBeUndefined(); + expect(result.error.message).not.toContain("invalidated"); + } finally { + writer.mockRestore(); + } +}); + it("uses confirmed publication after the output owner's descriptor cleanup completes", async () => { const root = await createTestTempDirectory("rea-bundle-published-"); const files = await writeEnvironmentServeFiles(root); diff --git a/src/application/environment/environmentBundleWrite.ts b/src/application/environment/environmentBundleWrite.ts index 06e7296b7d..d3ec3c55dc 100644 --- a/src/application/environment/environmentBundleWrite.ts +++ b/src/application/environment/environmentBundleWrite.ts @@ -10,6 +10,7 @@ import { import { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js"; import { ArtifactFileCleanupFailure } from "../../artifacts/ArtifactFileOperation.js"; import { RegularFileCleanupFailure } from "../../filesystem/OwnedRegularFile.js"; +import { readFileHandleChunks } from "../../filesystem/readFileHandleChunks.js"; import { streamChunkToBuffer } from "../../artifacts/StreamBytes.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js"; @@ -51,10 +52,7 @@ const writeTarball = async ( ): Promise => { const hash = createHash("sha256"); let observed = 0; - for await (const chunk of source.handle.createReadStream({ - start: 0, - autoClose: false, - })) { + for await (const chunk of readFileHandleChunks(source.handle)) { const bytes = streamChunkToBuffer(chunk); observed += bytes.length; if (observed > source.size) throw new RegularFileChangedError(source.path); @@ -63,7 +61,7 @@ const writeTarball = async ( if (observed !== source.size) throw new RegularFileChangedError(source.path); const written = await tree.write( relative, - source.handle.createReadStream({ start: 0, autoClose: false }), + readFileHandleChunks(source.handle), { bytes: source.size, sha256: hash.digest("hex"), From 1bb2fa23de310bf334281b17eb0f5636888a1422 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:50:50 +0800 Subject: [PATCH 19/35] test(apple): match plist provenance to the selected workflow --- scripts/verify/apple/native-provenance.mjs | 67 +++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/scripts/verify/apple/native-provenance.mjs b/scripts/verify/apple/native-provenance.mjs index a0cf192a7e..63ae2183b0 100644 --- a/scripts/verify/apple/native-provenance.mjs +++ b/scripts/verify/apple/native-provenance.mjs @@ -6,6 +6,7 @@ import { toolContract } from "../../../dist/contracts/toolContracts.js"; import { artifactCliEvidence, withArtifactMcp, + withBundleResourcesMcp, } from "../../lib/artifact-e2e.mjs"; /** Inspect the compiled bundle with real native utilities through both adapters. */ @@ -16,15 +17,79 @@ export async function verifyNativeProvenance(app, executable) { const operations = [ ["inspect_macho", []], ["inspect_signature", []], - ["inspect_plist", []], ["list_architectures", []], ["demangle_swift", symbols], ]; const observations = []; + const resourcePlistCli = toolContract("inspect_plist").outputSchema.parse( + await artifactCliEvidence("inspect-plist", app), + ); + assert.equal(resourcePlistCli.subject, null); + assert.equal(resourcePlistCli.raw_result, null); + await withBundleResourcesMcp(app, async (client) => { + const response = await client.callTool({ + name: "inspect_plist", + arguments: {}, + }); + assert.notEqual(response.isError, true, JSON.stringify(response)); + const mcp = toolContract("inspect_plist").outputSchema.parse( + response.structuredContent, + ); + assert.equal(mcp.subject, null); + assert.equal(mcp.raw_result, null); + assert.deepEqual(mcp.normalized_result, resourcePlistCli.normalized_result); + assert.deepEqual( + JSON.parse(response.content.find((item) => item.type === "text").text), + response.structuredContent, + ); + await client.ping(); + observations.push({ + operation: "inspect_plist_resource_root", + commands: resourcePlistCli.normalized_result.provenance.length, + cli_bytes: Buffer.byteLength(JSON.stringify(resourcePlistCli)), + mcp_bytes: Buffer.byteLength(JSON.stringify(response)), + }); + }); + + const binaryPlistCli = toolContract("inspect_plist").outputSchema.parse( + await artifactCliEvidence("inspect-plist", executable), + ); + assert.equal(binaryPlistCli.subject?.digest.sha256, sha256); + assert.equal(binaryPlistCli.raw_result, null); + assert.equal(binaryPlistCli.normalized_result.resource_root, undefined); await withArtifactMcp(app, async (client) => { const definitions = new Map( (await client.listTools()).tools.map((tool) => [tool.name, tool]), ); + const plistResponse = await client.callTool( + { name: "inspect_plist", arguments: {} }, + { toolDefinition: definitions.get("inspect_plist") }, + ); + assert.notEqual(plistResponse.isError, true, JSON.stringify(plistResponse)); + const binaryPlistMcp = toolContract("inspect_plist").outputSchema.parse( + plistResponse.structuredContent, + ); + assert.equal(binaryPlistMcp.subject?.digest.sha256, sha256); + assert.equal(binaryPlistMcp.raw_result, null); + assert.equal(binaryPlistMcp.normalized_result.resource_root, undefined); + assert.deepEqual( + binaryPlistMcp.normalized_result, + binaryPlistCli.normalized_result, + ); + assert.deepEqual( + JSON.parse( + plistResponse.content.find((item) => item.type === "text").text, + ), + plistResponse.structuredContent, + ); + await client.ping(); + observations.push({ + operation: "inspect_plist_binary_bound", + commands: binaryPlistCli.normalized_result.provenance.length, + cli_bytes: Buffer.byteLength(JSON.stringify(binaryPlistCli)), + mcp_bytes: Buffer.byteLength(JSON.stringify(plistResponse)), + }); + for (const [operation, arguments_] of operations) { const cli = toolContract(operation).outputSchema.parse( await artifactCliEvidence( From 7dd6941393c852a3324cf01c2bd79a9dcff43728 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:50:50 +0800 Subject: [PATCH 20/35] refactor(test): share browser and apktool fixture lifecycles --- src/browser/IsolatedWebBundleAnalyzer.test.ts | 114 +---- src/browser/PlaywrightScenarioSession.test.ts | 121 +++--- src/domain/webBundleAnalyzer.test.ts | 113 +---- .../browser/cdpObservationWindowScope.test.ts | 102 ++--- .../boundary/browser/cdpSvgXlinkHref.test.ts | 388 ++++++++---------- .../cli/apktoolBooleanOptions.test.ts | 50 +-- .../boundary/cli/apktoolLocaleOption.test.ts | 54 +-- tests/support/browser/webPageInspection.ts | 122 ++++++ tests/support/cli/apktoolCli.ts | 44 ++ 9 files changed, 461 insertions(+), 647 deletions(-) create mode 100644 tests/support/browser/webPageInspection.ts create mode 100644 tests/support/cli/apktoolCli.ts diff --git a/src/browser/IsolatedWebBundleAnalyzer.test.ts b/src/browser/IsolatedWebBundleAnalyzer.test.ts index 131f839750..d4d5df8176 100644 --- a/src/browser/IsolatedWebBundleAnalyzer.test.ts +++ b/src/browser/IsolatedWebBundleAnalyzer.test.ts @@ -1,6 +1,6 @@ import { expect, it } from "vitest"; +import { createWebPageInspection } from "../../tests/support/browser/webPageInspection.js"; -import { webPageInspectionSchema } from "../domain/browserObservationSchemas.js"; import { AnalysisCancelledError, AnalysisOutputError, @@ -13,7 +13,6 @@ import type { } from "../domain/webBundleAnalysis.js"; import { analyzeWebBundleInputSchema } from "../domain/webBundleAnalysis.js"; import { analyzeCapturedWebScript } from "../domain/webBundleAnalyzer.js"; -import { createWebTextArtifact } from "../domain/webContentArtifact.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok } from "../domain/result.js"; import { @@ -373,107 +372,12 @@ const deferred = (): { }; const inspection = (sources: readonly string[]) => - webPageInspectionSchema.parse({ - browser: { - product: "Fake Chrome", - protocol_version: "1.3", - revision: "1", - user_agent: "fake", - js_version: "1", - }, - target: { - target_id: "page-1", - type: "page", - title: "App", - url: `${origin}/app`, + createWebPageInspection( + sources.map((source, index) => ({ + source, + scriptKey: `scr_${String(index + 1).repeat(64)}`, + url: `${origin}/assets/${String(index)}.js`, origin, - attached: false, - }, - capture_window: { - started_at: "2026-07-14T00:00:00.000Z", - ended_at: "2026-07-14T00:00:01.000Z", - observation_ms: 1_000, - }, - completeness: { - status: "attach_limited", - conditions: ["attach_limited"], - policy_filtered_sections: [], - attach_limited_sections: ["network_requests"], - truncated_sections: [], - unavailable_sections: [], - excluded: [], - dropped_events: { - scripts: 0, - network_requests: 0, - console_events: 0, - websocket_connections: 0, - websocket_frames: 0, - webmcp_tools: 0, - timeline_events: 0, - total: 0, - }, - }, - frames: [], - dom: { total_nodes: 0, nodes: [] }, - accessibility: { - total_nodes: 0, - text_capture: { - status: "not_approved", - retained_bytes: 0, - excluded_fields: 0, - }, - nodes: [], - }, - scripts: { - total: sources.length, - items: sources.map((source, index) => ({ - script_key: `scr_${String(index + 1).repeat(64)}`, - url: `${origin}/assets/${String(index)}.js`, - origin, - cdp_hash: `hash-${String(index)}`, - length: Buffer.byteLength(source), - is_module: true, - language: "JavaScript", - source_map_url: null, - resource_reconciliation: { - status: "unmatched", - reason: "no_exact_sanitized_url", - }, - source: { - included: true, - artifact: createWebTextArtifact(source, "text/javascript"), - }, - })), - }, - resources: [], - network: { - requests: [], - websocket_connections: [], - coverage_started_at: "2026-07-14T00:00:00.000Z", - prior_activity_available: false, - }, - console: { - events: [], - coverage_started_at: "2026-07-14T00:00:00.000Z", - prior_activity_available: false, - }, - workers: [], - metadata: { - responses: [], - dom_urls: [], - agent_hints: [], - excluded_dom_urls: 0, - headers_allowlisted: true, - }, - storage: { - origin, - usage_bytes: null, - quota_bytes: null, - local_storage_keys: [], - session_storage_keys: [], - indexed_db_names: [], - cache_names: [], - values_redacted: true, - }, - limitations: [], - }); + cdpHash: `hash-${String(index)}`, + })), + ); diff --git a/src/browser/PlaywrightScenarioSession.test.ts b/src/browser/PlaywrightScenarioSession.test.ts index 04125b3ddb..7615c27dd6 100644 --- a/src/browser/PlaywrightScenarioSession.test.ts +++ b/src/browser/PlaywrightScenarioSession.test.ts @@ -216,6 +216,51 @@ const policyResponse = (method: string): unknown => ? { result: { value: null } } : {}; +const prepareLateStopRetry = async () => { + const { cleanup, page, storage } = fixture(); + const cancellation = new AbortController(); + storage?.close.mockResolvedValue({ loadingStopped: false }); + const attached = connectedScenario("block"); + const session = await PlaywrightScenarioSession.open( + attached, + {}, + { signal: cancellation.signal }, + ); + const { promise: held, resolve: finishRestoration } = deferred(); + const { promise: started, resolve: restorationStarted } = deferred(); + const { promise: transportClosed, resolve: signalTransportClosed } = + deferred(); + mocks.policyTransportClose.mockImplementation(async () => { + signalTransportClosed(); + }); + mocks.policySend.mockImplementation(async (method: string) => { + if (method === "Runtime.evaluate") { + restorationStarted(); + await held; + } + return policyResponse(method); + }); + + vi.useFakeTimers(); + try { + const firstClose = session.close(false).then( + () => ({ ok: true as const }), + (cause: unknown) => ({ ok: false as const, cause }), + ); + await started; + await vi.advanceTimersByTimeAsync(5_001); + await expect(firstClose).resolves.toMatchObject({ ok: false }); + finishRestoration(); + await transportClosed; + await vi.advanceTimersByTimeAsync(0); + cancellation.abort(); + return { cleanup, page, session }; + } catch (cause) { + vi.useRealTimers(); + throw cause; + } +}; + beforeEach(() => { vi.clearAllMocks(); mocks.policySend.mockReset(); @@ -661,44 +706,8 @@ it("retries a failed connected-page stop when service workers are allowed", asyn }); it("retains a late fallback stop failure for retry", async () => { - const { cleanup, page, storage } = fixture(); - const cancellation = new AbortController(); - storage?.close.mockResolvedValue({ loadingStopped: false }); - const attached = connectedScenario("block"); - const session = await PlaywrightScenarioSession.open( - attached, - {}, - { signal: cancellation.signal }, - ); - const { promise: held, resolve: finishRestoration } = deferred(); - const { promise: started, resolve: restorationStarted } = deferred(); - const { promise: transportClosed, resolve: signalTransportClosed } = - deferred(); - mocks.policyTransportClose.mockImplementation(async () => { - signalTransportClosed(); - }); - mocks.policySend.mockImplementation(async (method: string) => { - if (method === "Runtime.evaluate") { - restorationStarted(); - await held; - } - return policyResponse(method); - }); - - vi.useFakeTimers(); + const { cleanup, page, session } = await prepareLateStopRetry(); try { - const firstClose = session.close(false).then( - () => ({ ok: true as const }), - (cause: unknown) => ({ ok: false as const, cause }), - ); - await started; - await vi.advanceTimersByTimeAsync(5_001); - await expect(firstClose).resolves.toMatchObject({ ok: false }); - finishRestoration(); - await transportClosed; - await vi.advanceTimersByTimeAsync(0); - - cancellation.abort(); const stopFailure = new Error("late main-frame stop failed"); const detachedFrame = { evaluate: vi.fn(async () => { @@ -734,44 +743,8 @@ it("retains a late fallback stop failure for retry", async () => { }); it("coalesces concurrent late-stop close retries", async () => { - const { cleanup, page, storage } = fixture(); - const cancellation = new AbortController(); - storage?.close.mockResolvedValue({ loadingStopped: false }); - const attached = connectedScenario("block"); - const session = await PlaywrightScenarioSession.open( - attached, - {}, - { signal: cancellation.signal }, - ); - const { promise: held, resolve: finishRestoration } = deferred(); - const { promise: started, resolve: restorationStarted } = deferred(); - const { promise: transportClosed, resolve: signalTransportClosed } = - deferred(); - mocks.policyTransportClose.mockImplementation(async () => { - signalTransportClosed(); - }); - mocks.policySend.mockImplementation(async (method: string) => { - if (method === "Runtime.evaluate") { - restorationStarted(); - await held; - } - return policyResponse(method); - }); - - vi.useFakeTimers(); + const { cleanup, page, session } = await prepareLateStopRetry(); try { - const firstClose = session.close(false).then( - () => ({ ok: true as const }), - (cause: unknown) => ({ ok: false as const, cause }), - ); - await started; - await vi.advanceTimersByTimeAsync(5_001); - await expect(firstClose).resolves.toMatchObject({ ok: false }); - finishRestoration(); - await transportClosed; - await vi.advanceTimersByTimeAsync(0); - - cancellation.abort(); const { promise: heldStop, resolve: finishStop } = deferred(); const { promise: startedStop, resolve: stopStarted } = deferred(); page.evaluate.mockImplementation(async (script: unknown) => { diff --git a/src/domain/webBundleAnalyzer.test.ts b/src/domain/webBundleAnalyzer.test.ts index f7af053d30..c8b5f820fa 100644 --- a/src/domain/webBundleAnalyzer.test.ts +++ b/src/domain/webBundleAnalyzer.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; +import { createWebPageInspection } from "../../tests/support/browser/webPageInspection.js"; -import { webPageInspectionSchema } from "./browserObservationSchemas.js"; import { analyzeJavaScriptStaticSource } from "./javascript/javascriptStaticAnalysis.js"; import { analyzeCapturedWebBundle, @@ -1146,112 +1146,15 @@ describe("deep captured web bundle syntax", () => { }); const inspection = (source: string) => - webPageInspectionSchema.parse({ - browser: { - product: "Fake Chrome", - protocol_version: "1.3", - revision: "1", - user_agent: "fake", - js_version: "1", - }, - target: { - target_id: "page-1", - type: "page", - title: "App", - url: `${origin}/app`, + createWebPageInspection([ + { + source, + scriptKey: `scr_${"1".repeat(64)}`, + url: `${origin}/assets/app.js`, origin, - attached: false, + cdpHash: "hash", }, - capture_window: { - started_at: "2026-07-14T00:00:00.000Z", - ended_at: "2026-07-14T00:00:01.000Z", - observation_ms: 1_000, - }, - completeness: { - status: "attach_limited", - conditions: ["attach_limited"], - policy_filtered_sections: [], - attach_limited_sections: ["network_requests"], - truncated_sections: [], - unavailable_sections: [], - excluded: [], - dropped_events: { - scripts: 0, - network_requests: 0, - console_events: 0, - websocket_connections: 0, - websocket_frames: 0, - webmcp_tools: 0, - timeline_events: 0, - total: 0, - }, - }, - frames: [], - dom: { total_nodes: 0, nodes: [] }, - accessibility: { - total_nodes: 0, - text_capture: { - status: "not_approved", - retained_bytes: 0, - excluded_fields: 0, - }, - nodes: [], - }, - scripts: { - total: 1, - items: [ - { - script_key: `scr_${"1".repeat(64)}`, - url: `${origin}/assets/app.js`, - origin, - cdp_hash: "hash", - length: Buffer.byteLength(source), - is_module: true, - language: "JavaScript", - source_map_url: null, - resource_reconciliation: { - status: "unmatched", - reason: "no_exact_sanitized_url", - }, - source: { - included: true, - artifact: createWebTextArtifact(source, "text/javascript"), - }, - }, - ], - }, - resources: [], - network: { - requests: [], - websocket_connections: [], - coverage_started_at: "2026-07-14T00:00:00.000Z", - prior_activity_available: false, - }, - console: { - events: [], - coverage_started_at: "2026-07-14T00:00:00.000Z", - prior_activity_available: false, - }, - workers: [], - metadata: { - responses: [], - dom_urls: [], - agent_hints: [], - excluded_dom_urls: 0, - headers_allowlisted: true, - }, - storage: { - origin, - usage_bytes: null, - quota_bytes: null, - local_storage_keys: [], - session_storage_keys: [], - indexed_db_names: [], - cache_names: [], - values_redacted: true, - }, - limitations: [], - }); + ]); describe("per-script web bundle analysis", () => { it("keeps parser failures as analyzed outcomes with their own coverage", () => { diff --git a/tests/boundary/browser/cdpObservationWindowScope.test.ts b/tests/boundary/browser/cdpObservationWindowScope.test.ts index 9d5f44a965..c3b8e5f10c 100644 --- a/tests/boundary/browser/cdpObservationWindowScope.test.ts +++ b/tests/boundary/browser/cdpObservationWindowScope.test.ts @@ -2,17 +2,39 @@ import { expect, it } from "vitest"; import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; import { observeWebSessionInputSchema } from "../../../src/domain/browserSession.js"; -import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; +import { + startFakeCdpBrowser, + type FakeOptions, +} from "../../fixtures/fakeCdpBrowser.js"; import { trackBrowser } from "./cdpBrowserProvider.support.js"; -it("fails closed on an untimestamped cross-origin main-frame commit during clock sampling", async () => { +const observeDuringClockSampling = async ( + commandEvents: NonNullable, +) => { const browser = await startFakeCdpBrowser({ eventsBeforeReply: ["Performance.getMetrics"], commandResult: (command) => command.method === "Performance.getMetrics" ? { metrics: [{ name: "Timestamp", value: 20 }] } : undefined, - commandEvents: (command) => { + commandEvents, + }); + trackBrowser(browser); + const result = await new CdpBrowserProvider().observeSession( + observeWebSessionInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 20, + }), + ); + if (!result.ok) throw result.error; + return { browser, observation: result.value }; +}; + +it("fails closed on an untimestamped cross-origin main-frame commit during clock sampling", async () => { + const { observation } = await observeDuringClockSampling( + (command, origin) => { if (command.method !== "Performance.getMetrics") return undefined; return [ { @@ -34,27 +56,17 @@ it("fails closed on an untimestamped cross-origin main-frame commit during clock timestamp: 21, type: "Document", frameId: "frame-main", - redirectResponse: { url: `${browser.allowedOrigin}/prior` }, - request: { url: `${browser.allowedOrigin}/later` }, + redirectResponse: { url: `${origin}/prior` }, + request: { url: `${origin}/later` }, }, }, ]; }, - }); - trackBrowser(browser); - const result = await new CdpBrowserProvider().observeSession( - observeWebSessionInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 20, - }), ); - if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("target_left_scope"); - expect(result.value.target.final_url).toBeNull(); - expect(result.value.timeline).toContainEqual( + expect(observation.window.end_reason).toBe("target_left_scope"); + expect(observation.target.final_url).toBeNull(); + expect(observation.timeline).toContainEqual( expect.objectContaining({ type: "navigation_committed", loader_id: "outside-loader", @@ -63,19 +75,14 @@ it("fails closed on an untimestamped cross-origin main-frame commit during clock url: null, }), ); - expect(result.value.timeline).not.toContainEqual( + expect(observation.timeline).not.toContainEqual( expect.objectContaining({ request_id: "later-in-window-request" }), ); }); it("fails closed on an untimestamped out-of-scope main-frame request during clock sampling", async () => { - const browser = await startFakeCdpBrowser({ - eventsBeforeReply: ["Performance.getMetrics"], - commandResult: (command) => - command.method === "Performance.getMetrics" - ? { metrics: [{ name: "Timestamp", value: 20 }] } - : undefined, - commandEvents: (command, origin) => { + const { observation } = await observeDuringClockSampling( + (command, origin) => { if (command.method !== "Performance.getMetrics") return undefined; return [ { @@ -101,20 +108,10 @@ it("fails closed on an untimestamped out-of-scope main-frame request during cloc }, ]; }, - }); - trackBrowser(browser); - const result = await new CdpBrowserProvider().observeSession( - observeWebSessionInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 20, - }), ); - if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("target_left_scope"); - expect(result.value.timeline).toContainEqual( + expect(observation.window.end_reason).toBe("target_left_scope"); + expect(observation.timeline).toContainEqual( expect.objectContaining({ type: "navigation_requested", timestamp: null, @@ -122,19 +119,14 @@ it("fails closed on an untimestamped out-of-scope main-frame request during cloc url: null, }), ); - expect(result.value.timeline).not.toContainEqual( + expect(observation.timeline).not.toContainEqual( expect.objectContaining({ request_id: "later-in-window-request" }), ); }); it("fails closed on an untimestamped out-of-scope redirect during clock sampling", async () => { - const browser = await startFakeCdpBrowser({ - eventsBeforeReply: ["Performance.getMetrics"], - commandResult: (command) => - command.method === "Performance.getMetrics" - ? { metrics: [{ name: "Timestamp", value: 20 }] } - : undefined, - commandEvents: (command, origin) => { + const { observation } = await observeDuringClockSampling( + (command, origin) => { if (command.method !== "Performance.getMetrics") return undefined; return [ { @@ -162,21 +154,11 @@ it("fails closed on an untimestamped out-of-scope redirect during clock sampling }, ]; }, - }); - trackBrowser(browser); - const result = await new CdpBrowserProvider().observeSession( - observeWebSessionInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 20, - }), ); - if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("target_left_scope"); - expect(result.value.target.final_url).toBeNull(); - expect(result.value.timeline).toContainEqual( + expect(observation.window.end_reason).toBe("target_left_scope"); + expect(observation.target.final_url).toBeNull(); + expect(observation.timeline).toContainEqual( expect.objectContaining({ type: "redirect", request_id: "unknown-time-outside-redirect", @@ -185,7 +167,7 @@ it("fails closed on an untimestamped out-of-scope redirect during clock sampling url: null, }), ); - expect(result.value.timeline).not.toContainEqual( + expect(observation.timeline).not.toContainEqual( expect.objectContaining({ request_id: "later-in-window-request" }), ); }); diff --git a/tests/boundary/browser/cdpSvgXlinkHref.test.ts b/tests/boundary/browser/cdpSvgXlinkHref.test.ts index 38048861b6..7d1e2a6274 100644 --- a/tests/boundary/browser/cdpSvgXlinkHref.test.ts +++ b/tests/boundary/browser/cdpSvgXlinkHref.test.ts @@ -14,6 +14,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { processTest as it, waitForExit, + type TestProcesses, } from "../../support/process/processFixture.js"; const chromeExecutable = [ @@ -34,6 +35,87 @@ const chromeExecutable = [ const CHROME_STARTUP_TIMEOUT_MS = 60_000; const CHROME_TEST_TIMEOUT_MS = CHROME_STARTUP_TIMEOUT_MS + 30_000; +const inspectChromePage = async ({ + processes, + executable, + profilePrefix, + page, + contentType, + pathname, +}: { + processes: TestProcesses; + executable: string; + profilePrefix: string; + page: string; + contentType: string; + pathname: string; +}) => { + const profile = await createTestTempDirectory(profilePrefix); + const server = createServer((_request, response) => { + response.writeHead(200, { + "content-type": `${contentType}; charset=utf-8`, + }); + response.end(page); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("SVG fixture server did not bind a TCP port"); + const origin = `http://127.0.0.1:${String(address.port)}`; + const pageUrl = `${origin}${pathname}`; + let stderr = ""; + const child = processes.spawn(executable, [ + "--headless=new", + ...(process.env.REA_BROWSER_NO_SANDBOX === "true" ? ["--no-sandbox"] : []), + "--remote-debugging-address=127.0.0.1", + "--remote-debugging-port=0", + `--user-data-dir=${profile}`, + "--no-first-run", + "--no-default-browser-check", + "--disable-background-networking", + "--disable-component-update", + "--disable-dev-shm-usage", + "--disable-sync", + "--metrics-recording-only", + pageUrl, + ]); + child.stdout.on("data", () => undefined); + child.stderr.on("data", (chunk: Buffer) => { + if (stderr.length < 8_192) stderr += chunk.toString("utf8"); + }); + try { + const port = await waitForBrowserDevtoolsPort({ + child, + executable, + activePortPath: join(profile, "DevToolsActivePort"), + stderr: () => stderr, + timeoutMs: CHROME_STARTUP_TIMEOUT_MS, + }); + const endpoint = `http://127.0.0.1:${String(port)}`; + const provider = new CdpBrowserProvider(); + const targetId = await pageTarget(provider, endpoint, origin, pageUrl); + const result = await provider.inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [origin], + target_id: targetId, + observation_ms: 0, + }), + ); + if (!result.ok) throw result.error; + return { origin, pageUrl, inspection: result.value }; + } finally { + child.kill("SIGTERM"); + if (!(await waitForExit(child, 2_000))) child.kill("SIGKILL"); + await waitForExit(child, 2_000); + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } +}; + // Issue 1586's page, plus the HTML, image, use, and foreign-origin cases the // real snapshot must distinguish. The XHTML page is the prefixed-svg gap // reported on PR 1602. The tests are skipped when no Chrome-family executable @@ -84,133 +166,78 @@ it.skipIf(chromeExecutable === undefined)( async ({ processes }) => { const executable = chromeExecutable; if (executable === undefined) return; - const profile = await createTestTempDirectory("rea-svg-xlink-"); - const server = createServer((_request, response) => { - response.writeHead(200, { "content-type": "text/html; charset=utf-8" }); - response.end(pageHtml); + const { origin, inspection } = await inspectChromePage({ + processes, + executable, + profilePrefix: "rea-svg-xlink-", + page: pageHtml, + contentType: "text/html", + pathname: "/x.html", }); - await new Promise((resolve) => { - server.listen(0, "127.0.0.1", resolve); - }); - const address = server.address(); - if (address === null || typeof address === "string") - throw new Error("SVG fixture server did not bind a TCP port"); - const origin = `http://127.0.0.1:${String(address.port)}`; - const pageUrl = `${origin}/x.html`; - let stderr = ""; - const child = processes.spawn(executable, [ - "--headless=new", - ...(process.env.REA_BROWSER_NO_SANDBOX === "true" - ? ["--no-sandbox"] - : []), - "--remote-debugging-address=127.0.0.1", - "--remote-debugging-port=0", - `--user-data-dir=${profile}`, - "--no-first-run", - "--no-default-browser-check", - "--disable-background-networking", - "--disable-component-update", - "--disable-dev-shm-usage", - "--disable-sync", - "--metrics-recording-only", - pageUrl, - ]); - child.stdout.on("data", () => undefined); - child.stderr.on("data", (chunk: Buffer) => { - if (stderr.length < 8_192) stderr += chunk.toString("utf8"); - }); - try { - const port = await waitForBrowserDevtoolsPort({ - child, - executable, - activePortPath: join(profile, "DevToolsActivePort"), - stderr: () => stderr, - timeoutMs: CHROME_STARTUP_TIMEOUT_MS, - }); - const endpoint = `http://127.0.0.1:${String(port)}`; - const provider = new CdpBrowserProvider(); - const targetId = await pageTarget(provider, endpoint, origin, pageUrl); - const result = await provider.inspectPage( - inspectWebPageInputSchema.parse({ - cdp_endpoint: endpoint, - allowed_origins: [origin], - target_id: targetId, - observation_ms: 0, + const urls = inspection.metadata.dom_urls; + const html = inspection.dom.nodes.find( + (node) => + node.node_name === "A" && node.attribute_names.includes("xlink:href"), + ); + expect(html).toBeDefined(); + expect(urls.some((url) => url.node_index === html?.index)).toBe(false); + expect(urls).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + attribute: "href", + url: `${origin}/xl-s2h`, + destination_scope: "approved", }), - ); - if (!result.ok) throw result.error; - const urls = result.value.metadata.dom_urls; - const html = result.value.dom.nodes.find( - (node) => - node.node_name === "A" && node.attribute_names.includes("xlink:href"), - ); - expect(html).toBeDefined(); - expect(urls.some((url) => url.node_index === html?.index)).toBe(false); - expect(urls).toEqual( - expect.arrayContaining([ + expect.objectContaining({ + attribute: "href", + url: `${origin}/href-wins`, + destination_scope: "approved", + }), + expect.objectContaining({ + attribute: "href", + url: `${origin}/xl-image`, + destination_scope: "approved", + }), + expect.objectContaining({ + attribute: "href", + url: `${origin}/xl-use`, + destination_scope: "approved", + }), + expect.objectContaining({ + attribute: "href", + url: null, + destination_scope: "outside_policy", + }), + ...[ + "/html-desc", + "/html-title", + "/xl-nested-desc", + "/xl-nested-title", + "/xl-moved-desc", + "/xl-moved-title", + "/xl-created-desc", + "/xl-created-title", + "/xl-created-nested-desc", + "/xl-created-nested-title", + ].map((path) => expect.objectContaining({ attribute: "href", - url: `${origin}/xl-s2h`, + url: `${origin}${path}`, destination_scope: "approved", }), - expect.objectContaining({ - attribute: "href", - url: `${origin}/href-wins`, - destination_scope: "approved", - }), - expect.objectContaining({ - attribute: "href", - url: `${origin}/xl-image`, - destination_scope: "approved", - }), - expect.objectContaining({ - attribute: "href", - url: `${origin}/xl-use`, - destination_scope: "approved", - }), - expect.objectContaining({ - attribute: "href", - url: null, - destination_scope: "outside_policy", - }), - ...[ - "/html-desc", - "/html-title", - "/xl-nested-desc", - "/xl-nested-title", - "/xl-moved-desc", - "/xl-moved-title", - "/xl-created-desc", - "/xl-created-title", - "/xl-created-nested-desc", - "/xl-created-nested-title", - ].map((path) => - expect.objectContaining({ - attribute: "href", - url: `${origin}${path}`, - destination_scope: "approved", - }), - ), - ]), - ); - expect( - urls.some( - (url) => - url.url?.includes("/xl-html") === true || - url.url?.includes("/xl-math") === true || - url.url?.endsWith("/xlink-loses") === true, ), - ).toBe(false); - expect(urls.every((url) => url.attribute === "href")).toBe(true); - expect(urls).toHaveLength(15); - } finally { - child.kill("SIGTERM"); - if (!(await waitForExit(child, 2_000))) child.kill("SIGKILL"); - await waitForExit(child, 2_000); - await new Promise((resolve) => { - server.close(() => resolve()); - }); - } + ]), + ); + expect( + urls.some( + (url) => + url.url?.includes("/xl-html") === true || + url.url?.includes("/xl-math") === true || + url.url?.endsWith("/xlink-loses") === true, + ), + ).toBe(false); + expect(urls.every((url) => url.attribute === "href")).toBe(true); + expect(urls).toHaveLength(15); }, CHROME_TEST_TIMEOUT_MS, ); @@ -277,99 +304,42 @@ for (const { format, page, contentType, nodeNames, link, destinations } of [ async ({ processes }) => { const executable = chromeExecutable; if (executable === undefined) return; - const profile = await createTestTempDirectory("rea-svg-xhtml-"); - const server = createServer((_request, response) => { - response.writeHead(200, { - "content-type": `${contentType}; charset=utf-8`, - }); - response.end(page); + const { origin, pageUrl, inspection } = await inspectChromePage({ + processes, + executable, + profilePrefix: "rea-svg-xhtml-", + page, + contentType, + pathname: "/page.xhtml", }); - await new Promise((resolve) => { - server.listen(0, "127.0.0.1", resolve); - }); - const address = server.address(); - if (address === null || typeof address === "string") - throw new Error("XHTML fixture server did not bind a TCP port"); - const origin = `http://127.0.0.1:${String(address.port)}`; - const pageUrl = `${origin}/page.xhtml`; - let stderr = ""; - const child = processes.spawn(executable, [ - "--headless=new", - ...(process.env.REA_BROWSER_NO_SANDBOX === "true" - ? ["--no-sandbox"] - : []), - "--remote-debugging-address=127.0.0.1", - "--remote-debugging-port=0", - `--user-data-dir=${profile}`, - "--no-first-run", - "--no-default-browser-check", - "--disable-background-networking", - "--disable-component-update", - "--disable-dev-shm-usage", - "--disable-sync", - "--metrics-recording-only", - pageUrl, - ]); - child.stdout.on("data", () => undefined); - child.stderr.on("data", (chunk: Buffer) => { - if (stderr.length < 8_192) stderr += chunk.toString("utf8"); - }); - try { - const port = await waitForBrowserDevtoolsPort({ - child, - executable, - activePortPath: join(profile, "DevToolsActivePort"), - stderr: () => stderr, - timeoutMs: CHROME_STARTUP_TIMEOUT_MS, - }); - const endpoint = `http://127.0.0.1:${String(port)}`; - const provider = new CdpBrowserProvider(); - const targetId = await pageTarget(provider, endpoint, origin, pageUrl); - const result = await provider.inspectPage( - inspectWebPageInputSchema.parse({ - cdp_endpoint: endpoint, - allowed_origins: [origin], - target_id: targetId, - observation_ms: 0, - }), - ); - if (!result.ok) throw result.error; - const urls = result.value.metadata.dom_urls; - for (const name of nodeNames) { - expect( - result.value.dom.nodes.some((node) => node.node_name === name), - ).toBe(true); - } + const urls = inspection.metadata.dom_urls; + for (const name of nodeNames) { expect( - result.value.dom.nodes.some( - (node) => - node.node_name === link.name && - node.attribute_names.includes(link.attribute), - ), + inspection.dom.nodes.some((node) => node.node_name === name), ).toBe(true); - expect(urls).toEqual( - destinations.map((destination) => - expect.objectContaining({ - attribute: "href", - url: - destination === null - ? null - : destination === "" - ? pageUrl - : `${origin}${destination}`, - destination_scope: - destination === null ? "outside_policy" : "approved", - }), - ), - ); - } finally { - child.kill("SIGTERM"); - if (!(await waitForExit(child, 2_000))) child.kill("SIGKILL"); - await waitForExit(child, 2_000); - await new Promise((resolve) => { - server.close(() => resolve()); - }); } + expect( + inspection.dom.nodes.some( + (node) => + node.node_name === link.name && + node.attribute_names.includes(link.attribute), + ), + ).toBe(true); + expect(urls).toEqual( + destinations.map((destination) => + expect.objectContaining({ + attribute: "href", + url: + destination === null + ? null + : destination === "" + ? pageUrl + : `${origin}${destination}`, + destination_scope: + destination === null ? "outside_policy" : "approved", + }), + ), + ); }, CHROME_TEST_TIMEOUT_MS, ); diff --git a/tests/boundary/cli/apktoolBooleanOptions.test.ts b/tests/boundary/cli/apktoolBooleanOptions.test.ts index 5c5578c18a..9da4353112 100644 --- a/tests/boundary/cli/apktoolBooleanOptions.test.ts +++ b/tests/boundary/cli/apktoolBooleanOptions.test.ts @@ -1,51 +1,9 @@ import { expect, it } from "vitest"; -import { Cli } from "incur"; -import pino from "pino"; -import type { ApktoolResourceAnalysisPort } from "../../../src/application/apktool/ApktoolResourceAnalysisPort.js"; -import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; -import { registerApktoolCommands } from "../../../src/cli/apktoolCommands.js"; -import { presentOmittableDefaults } from "../../../src/cli/omittableDefaults.js"; -import type { ApktoolRequest } from "../../../src/domain/apktool/apktoolResourceAnalysis.js"; -import { ok } from "../../../src/domain/result.js"; - -const invoke = async (argv: readonly string[]) => { - const requests: ApktoolRequest[] = []; - const provider: ApktoolResourceAnalysisPort = { - async inspectAvailability() { - throw new Error("unexpected provider probe"); - }, - async close() {}, - async execute(request) { - requests.push(request); - return ok( - createAnalysisExecution( - { strings: [] }, - { id: "apktool", name: "Apktool", version: null }, - ), - ); - }, - }; - const cli = Cli.create("rea", { sync: false }); - registerApktoolCommands(cli, pino({ level: "silent" }), {}, provider); - presentOmittableDefaults(cli); - - let stdout = ""; - let exitCode = 0; - await cli.serve([...argv], { - env: {}, - exit: (code) => { - exitCode = code; - }, - stdout: (text) => { - stdout += text; - }, - }); - return { exitCode, stdout, requests }; -}; +import { invokeApktoolCli } from "../../support/cli/apktoolCli.js"; it("dispatches --no-strings as include_strings false", async () => { - const result = await invoke([ + const result = await invokeApktoolCli([ "decode-android-resources", "/targets/app.apk", "--no-strings", @@ -62,7 +20,7 @@ it("dispatches --no-strings as include_strings false", async () => { }); it("keeps string projection enabled when the flag is omitted", async () => { - const result = await invoke([ + const result = await invokeApktoolCli([ "decode-android-resources", "/targets/app.apk", "--json", @@ -78,7 +36,7 @@ it("keeps string projection enabled when the flag is omitted", async () => { }); it("documents the supported negative flag in command help", async () => { - const result = await invoke(["decode-android-resources", "--help"]); + const result = await invokeApktoolCli(["decode-android-resources", "--help"]); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("--no-strings"); diff --git a/tests/boundary/cli/apktoolLocaleOption.test.ts b/tests/boundary/cli/apktoolLocaleOption.test.ts index 941b786cbc..65400be0b9 100644 --- a/tests/boundary/cli/apktoolLocaleOption.test.ts +++ b/tests/boundary/cli/apktoolLocaleOption.test.ts @@ -1,48 +1,6 @@ import { expect, it } from "vitest"; -import { Cli } from "incur"; -import pino from "pino"; -import type { ApktoolResourceAnalysisPort } from "../../../src/application/apktool/ApktoolResourceAnalysisPort.js"; -import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; -import { registerApktoolCommands } from "../../../src/cli/apktoolCommands.js"; -import { presentOmittableDefaults } from "../../../src/cli/omittableDefaults.js"; -import type { ApktoolRequest } from "../../../src/domain/apktool/apktoolResourceAnalysis.js"; -import { ok } from "../../../src/domain/result.js"; - -const invoke = async (argv: readonly string[]) => { - const requests: ApktoolRequest[] = []; - const provider: ApktoolResourceAnalysisPort = { - async inspectAvailability() { - throw new Error("unexpected provider probe"); - }, - async close() {}, - async execute(request) { - requests.push(request); - return ok( - createAnalysisExecution( - { strings: [] }, - { id: "apktool", name: "Apktool", version: null }, - ), - ); - }, - }; - const cli = Cli.create("rea", { sync: false }); - registerApktoolCommands(cli, pino({ level: "silent" }), {}, provider); - presentOmittableDefaults(cli); - - let stdout = ""; - let exitCode = 0; - await cli.serve([...argv], { - env: {}, - exit: (code) => { - exitCode = code; - }, - stdout: (text) => { - stdout += text; - }, - }); - return { exitCode, stdout, requests }; -}; +import { invokeApktoolCli } from "../../support/cli/apktoolCli.js"; /** * Locales the apktool input owner admits. The owner's own rejection message @@ -64,7 +22,7 @@ const OWNER_ADMITTED_LOCALES = [ ] as const; it("forwards the table-only string choice through the CLI", async () => { - const result = await invoke([ + const result = await invokeApktoolCli([ "decode-android-resources", "/targets/app.apk", "--string-tables-only", @@ -88,7 +46,7 @@ it("forwards the table-only string choice through the CLI", async () => { it("forwards every locale the apktool input owner admits", async () => { for (const locale of OWNER_ADMITTED_LOCALES) { - const result = await invoke([ + const result = await invokeApktoolCli([ "decode-android-resources", "/targets/app.apk", "--locale", @@ -107,7 +65,7 @@ it("forwards every locale the apktool input owner admits", async () => { }); it("forwards repeated --locales to project multiple localized tables", async () => { - const result = await invoke([ + const result = await invokeApktoolCli([ "decode-android-resources", "/targets/app.apk", "--locales", @@ -133,7 +91,7 @@ it("forwards repeated --locales to project multiple localized tables", async () it.each(["not a locale", "en-r419", "es-r419"])( "refuses the invalid locale %s before invoking Apktool", async (locale) => { - const result = await invoke([ + const result = await invokeApktoolCli([ "decode-android-resources", "/targets/app.apk", "--locale", @@ -151,7 +109,7 @@ it.each(["not a locale", "en-r419", "es-r419"])( ); it("forwards --no-strings with --locale preserving caller options", async () => { - const result = await invoke([ + const result = await invokeApktoolCli([ "decode-android-resources", "/targets/app.apk", "--no-strings", diff --git a/tests/support/browser/webPageInspection.ts b/tests/support/browser/webPageInspection.ts new file mode 100644 index 0000000000..e14124af0f --- /dev/null +++ b/tests/support/browser/webPageInspection.ts @@ -0,0 +1,122 @@ +import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js"; +import { createWebTextArtifact } from "../../../src/domain/webContentArtifact.js"; + +export interface WebPageScriptFixture { + source: string; + scriptKey: string; + url: string; + origin: string; + cdpHash: string; +} + +const origin = "https://app.example.test"; + +export const createWebPageInspection = ( + scripts: readonly WebPageScriptFixture[], +) => + webPageInspectionSchema.parse({ + browser: { + product: "Fake Chrome", + protocol_version: "1.3", + revision: "1", + user_agent: "fake", + js_version: "1", + }, + target: { + target_id: "page-1", + type: "page", + title: "App", + url: `${origin}/app`, + origin, + attached: false, + }, + capture_window: { + started_at: "2026-07-14T00:00:00.000Z", + ended_at: "2026-07-14T00:00:01.000Z", + observation_ms: 1_000, + }, + completeness: { + status: "attach_limited", + conditions: ["attach_limited"], + policy_filtered_sections: [], + attach_limited_sections: ["network_requests"], + truncated_sections: [], + unavailable_sections: [], + excluded: [], + dropped_events: { + scripts: 0, + network_requests: 0, + console_events: 0, + websocket_connections: 0, + websocket_frames: 0, + webmcp_tools: 0, + timeline_events: 0, + total: 0, + }, + }, + frames: [], + dom: { total_nodes: 0, nodes: [] }, + accessibility: { + total_nodes: 0, + text_capture: { + status: "not_approved", + retained_bytes: 0, + excluded_fields: 0, + }, + nodes: [], + }, + scripts: { + total: scripts.length, + items: scripts.map( + ({ source, scriptKey, url, origin: scriptOrigin, cdpHash }) => ({ + script_key: scriptKey, + url, + origin: scriptOrigin, + cdp_hash: cdpHash, + length: Buffer.byteLength(source), + is_module: true, + language: "JavaScript", + source_map_url: null, + resource_reconciliation: { + status: "unmatched", + reason: "no_exact_sanitized_url", + }, + source: { + included: true, + artifact: createWebTextArtifact(source, "text/javascript"), + }, + }), + ), + }, + resources: [], + network: { + requests: [], + websocket_connections: [], + coverage_started_at: "2026-07-14T00:00:00.000Z", + prior_activity_available: false, + }, + console: { + events: [], + coverage_started_at: "2026-07-14T00:00:00.000Z", + prior_activity_available: false, + }, + workers: [], + metadata: { + responses: [], + dom_urls: [], + agent_hints: [], + excluded_dom_urls: 0, + headers_allowlisted: true, + }, + storage: { + origin, + usage_bytes: null, + quota_bytes: null, + local_storage_keys: [], + session_storage_keys: [], + indexed_db_names: [], + cache_names: [], + values_redacted: true, + }, + limitations: [], + }); diff --git a/tests/support/cli/apktoolCli.ts b/tests/support/cli/apktoolCli.ts new file mode 100644 index 0000000000..cb3570ef60 --- /dev/null +++ b/tests/support/cli/apktoolCli.ts @@ -0,0 +1,44 @@ +import { Cli } from "incur"; +import pino from "pino"; + +import type { ApktoolResourceAnalysisPort } from "../../../src/application/apktool/ApktoolResourceAnalysisPort.js"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; +import { registerApktoolCommands } from "../../../src/cli/apktoolCommands.js"; +import { presentOmittableDefaults } from "../../../src/cli/omittableDefaults.js"; +import type { ApktoolRequest } from "../../../src/domain/apktool/apktoolResourceAnalysis.js"; +import { ok } from "../../../src/domain/result.js"; + +export const invokeApktoolCli = async (argv: readonly string[]) => { + const requests: ApktoolRequest[] = []; + const provider: ApktoolResourceAnalysisPort = { + async inspectAvailability() { + throw new Error("unexpected provider probe"); + }, + async close() {}, + async execute(request) { + requests.push(request); + return ok( + createAnalysisExecution( + { strings: [] }, + { id: "apktool", name: "Apktool", version: null }, + ), + ); + }, + }; + const cli = Cli.create("rea", { sync: false }); + registerApktoolCommands(cli, pino({ level: "silent" }), {}, provider); + presentOmittableDefaults(cli); + + let stdout = ""; + let exitCode = 0; + await cli.serve([...argv], { + env: {}, + exit: (code) => { + exitCode = code; + }, + stdout: (text) => { + stdout += text; + }, + }); + return { exitCode, stdout, requests }; +}; From b231c28cf64c2f799ada99343f8f21e0e6acb15e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:12:09 +0800 Subject: [PATCH 21/35] fix(hopper): own Mach-O preparation through cleanup retry Preserve exact selected-image identity and descriptor ownership across preparation, final cleanup, and retries. Keep physical failures and failed cleanup leases observable through provider and launcher boundaries. Closes #5688 --- scripts/verify-real-hopper-deadlines.mjs | 36 +- src/application/LinuxHopperPublisher.ts | 14 +- src/domain/apple/machoTargetSelection.ts | 79 ++ src/domain/hopperErrors.ts | 12 +- src/hopper/BridgeLauncher.ts | 128 ++- src/hopper/HopperAnalysisProfile.test.ts | 239 ++++- src/hopper/HopperAnalysisProfile.ts | 104 +- .../HopperMachOImage.copyLifecycle.test.ts | 342 ++++++ src/hopper/HopperMachOImage.test.ts | 404 ++++++- src/hopper/HopperMachOImage.ts | 996 ++++++++++++++---- src/hopper/HopperProvider.ts | 43 +- .../directAnalysisPermissions.test.ts | 6 + .../providers/hopper/bridgeLauncher.test.ts | 140 ++- .../hopper/hopperAnalysisProfile.test.ts | 7 + .../linuxHopperPublisherVerification.test.ts | 120 +-- 15 files changed, 2341 insertions(+), 329 deletions(-) create mode 100644 src/domain/apple/machoTargetSelection.ts create mode 100644 src/hopper/HopperMachOImage.copyLifecycle.test.ts diff --git a/scripts/verify-real-hopper-deadlines.mjs b/scripts/verify-real-hopper-deadlines.mjs index 11e0714314..31b2d856fc 100644 --- a/scripts/verify-real-hopper-deadlines.mjs +++ b/scripts/verify-real-hopper-deadlines.mjs @@ -6,6 +6,8 @@ import { fileURLToPath } from "node:url"; import { firstProcedureAddress } from "../dist/application/RealHopperAssertions.js"; import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; +import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js"; +import { AnalysisCleanupLease } from "../dist/artifacts/AnalysisCleanupLease.js"; import { resolveHopperAnalysisProfile } from "../dist/hopper/HopperAnalysisProfile.js"; import { HopperApplicationLauncher } from "../dist/hopper/BridgeLauncher.js"; import { HopperClient } from "../dist/hopper/HopperClient.js"; @@ -26,6 +28,16 @@ const root = await realpath( await mkdtemp(join(tmpdir(), "rea-hopper-deadlines-")), ); const target = join(root, "deadline-fixture"); +const profileResources = new ArtifactResourceScope(); +const profileResourceScope = { + retain: (owner, resource) => + profileResources.retain({ + kind: "lease", + resource, + lease: AnalysisCleanupLease.forCloseOwner(owner, resource), + }), + run: (operation) => profileResources.run(operation), +}; let client; let primaryFailure; try { @@ -39,6 +51,7 @@ try { launcherPath, loaderArgsOverride: [], provider: HOPPER_PROVIDER_IDENTITY, + resourceScope: profileResourceScope, }); if (!resolved.ok) throw resolved.error; const profile = resolved.value.profile; @@ -138,15 +151,28 @@ try { primaryFailure = cause; throw cause; } finally { + let cleanupFailure; if (client !== undefined) { const closed = await client.close(); - if (!closed.ok) - throw primaryFailure === undefined - ? closed.error + if (!closed.ok) cleanupFailure = closed.error; + } + try { + await profileResources.close(); + } catch (cause) { + cleanupFailure = + cleanupFailure === undefined + ? cause : new AggregateError( - [primaryFailure, closed.error], - "Hopper verification and cleanup failed", + [cleanupFailure, cause], + "Hopper profile and client cleanup failed", ); } await rm(root, { recursive: true, force: true }); + if (cleanupFailure !== undefined) + throw primaryFailure === undefined + ? cleanupFailure + : new AggregateError( + [primaryFailure, cleanupFailure], + "Hopper verification and cleanup failed", + ); } diff --git a/src/application/LinuxHopperPublisher.ts b/src/application/LinuxHopperPublisher.ts index 8acbefec98..eb63d578e1 100644 --- a/src/application/LinuxHopperPublisher.ts +++ b/src/application/LinuxHopperPublisher.ts @@ -7,7 +7,11 @@ import { ArtifactResourceScope, type ArtifactResourceOwner, } from "../artifacts/ArtifactResourceScope.js"; -import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; +import { + ArtifactReaderFailure, + artifactPrimaryCleanupLeases, +} from "../artifacts/ArtifactReader.js"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -1061,12 +1065,17 @@ export class LinuxHopperPublisher { export const retryLinuxHopperPublisherCleanup = async (): Promise => { const causes: unknown[] = []; const observations = []; + const cleanupLeases = new Set(); + let cleanupLeaseAuthorityKnown = true; for (const scope of pendingCleanup) { try { await scope.close(); pendingCleanup.delete(scope); } catch (cause: unknown) { causes.push(cause); + const leases = artifactPrimaryCleanupLeases(cause); + if (leases === undefined) cleanupLeaseAuthorityKnown = false; + else for (const lease of leases) cleanupLeases.add(lease); observations.push( ArtifactReaderFailure.cleanupObservation( cause, @@ -1088,6 +1097,9 @@ export const retryLinuxHopperPublisherCleanup = async (): Promise => { { cause: new AggregateError(causes, "Hopper publisher cleanup failures"), cleanup, + cleanupLeases: cleanupLeaseAuthorityKnown + ? [...cleanupLeases] + : undefined, }, ); } diff --git a/src/domain/apple/machoTargetSelection.ts b/src/domain/apple/machoTargetSelection.ts new file mode 100644 index 0000000000..f492bc2a0a --- /dev/null +++ b/src/domain/apple/machoTargetSelection.ts @@ -0,0 +1,79 @@ +import { z } from "zod"; + +import type { AnalysisProfileCommitment } from "../analysisProfile.js"; +import type { BinaryArchitecture } from "../binaryTargetTypes.js"; + +/** Provider-neutral observed identity of one Mach-O image selected from a source. */ +export const machoTargetSelectionSchema = z + .strictObject({ + kind: z.literal("macho-selected-image-v1"), + container: z.enum(["thin", "fat"]), + offset: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + byte_length: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + cpu_type: z.number().int().nonnegative().max(0xffff_ffff), + /** Keep the complete raw subtype, including capability bits. */ + cpu_subtype: z.number().int().nonnegative().max(0xffff_ffff), + container_table_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/u) + .nullable(), + }) + .superRefine((selection, context) => { + if ( + (selection.container === "thin" && + (selection.offset !== 0 || + selection.container_table_sha256 !== null)) || + (selection.container === "fat" && + selection.container_table_sha256 === null) + ) + context.addIssue({ + code: "custom", + message: "Invalid Mach-O container identity", + }); + }); + +export type MachoTargetSelection = z.infer; + +/** Read only the provider-neutral fact committed by an analysis profile. */ +export const profileMachoTargetSelection = ( + profile: AnalysisProfileCommitment | undefined, +): MachoTargetSelection | undefined => { + const raw = profile?.parameters.target_selection; + if (raw === undefined) return undefined; + const parsed = machoTargetSelectionSchema.safeParse(raw); + return parsed.success ? parsed.data : undefined; +}; + +/** Confirm the selected header's CPU family agrees with the caller's request. */ +export const machoTargetSelectionMatchesArchitecture = ( + selection: MachoTargetSelection, + architecture: BinaryArchitecture, +): boolean => { + const cpuType: Partial> = { + x86: 7, + x86_64: 0x01000007, + arm: 12, + arm64: 0x0100000c, + }; + return cpuType[architecture] === selection.cpu_type; +}; + +/** Confirm the parser-observed container agrees with the target's declared format. */ +export const machoTargetSelectionMatchesFormat = ( + selection: MachoTargetSelection, + format: "mach-o" | "mach-o-universal", +): boolean => selection.container === (format === "mach-o" ? "thin" : "fat"); + +/** Exact range and raw CPU identity equality; null is never treated as a match. */ +export const sameMachoTargetSelection = ( + left: MachoTargetSelection | undefined, + right: MachoTargetSelection | undefined, +): boolean => + left !== undefined && + right !== undefined && + left.container === right.container && + left.offset === right.offset && + left.byte_length === right.byte_length && + left.cpu_type === right.cpu_type && + left.cpu_subtype === right.cpu_subtype && + left.container_table_sha256 === right.container_table_sha256; diff --git a/src/domain/hopperErrors.ts b/src/domain/hopperErrors.ts index c5bb75c869..935dd1b48a 100644 --- a/src/domain/hopperErrors.ts +++ b/src/domain/hopperErrors.ts @@ -60,8 +60,14 @@ export class HopperTimeoutError extends HopperError { export class HopperCancelledError extends HopperError { readonly _tag = "HopperCancelledError"; - constructor() { - super("Hopper operation was cancelled"); + constructor( + cause?: unknown, + options: Omit = {}, + ) { + super( + "Hopper operation was cancelled", + cause === undefined ? options : { ...options, cause }, + ); } } @@ -167,7 +173,7 @@ export class HopperStartError extends HopperError { readonly launchRejection: HopperLaunchRejection | undefined; constructor( - options?: ErrorOptions & { + options?: AnalysisErrorOptions & { readonly userMessage?: string; readonly ownerRunId?: string; readonly launcherFailure?: HopperLauncherOutcome; diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 8e48b5e2a2..99431606f8 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -10,7 +10,13 @@ import { } from "../domain/hopperErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; -import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { + ArtifactReaderFailure, + artifactPrimaryCleanupLeases, +} from "../artifacts/ArtifactReader.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { AnalysisCleanupLeaseGroup } from "../artifacts/AnalysisCleanupLease.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js"; import { execFileOutput } from "../process/ExecFileOutput.js"; import { @@ -27,6 +33,7 @@ import { import { prepareHopperMachOImage, type HopperMachOImage, + type HopperMachOImageFileSystem, } from "./HopperMachOImage.js"; import { @@ -42,6 +49,19 @@ import { retryLinuxHopperPublisherCleanup } from "../application/LinuxHopperPubl const execFileAsync = promisify(execFile); const HOPPER_BACKGROUND_STARTUP_MS = 5_000; +const bindRetainedArtifactCleanup = ( + error: AnalysisError, + cause: unknown, +): void => { + const leases = artifactPrimaryCleanupLeases(cause); + if (leases !== undefined && leases.length > 0) { + new AnalysisCleanupLeaseGroup(error, leases); + return; + } + if (cause instanceof AnalysisError && cause.cleanupIncomplete) + error.bindCleanupDependencies([cause]); +}; + /** Coordinates for one private bridge session. */ export interface BridgeSession { readonly directory: string; @@ -115,6 +135,7 @@ export type HopperApplicationLauncherOptions = }); export interface HopperApplicationLauncherDependencies { + readonly imageFileSystem?: HopperMachOImageFileSystem; readonly verifyNativeHelper?: typeof verifyMacHopperNativeHelper; readonly verifyLinuxLauncher?: typeof inspectLinuxHopperLauncher; readonly retryPublisherCleanup?: typeof retryLinuxHopperPublisherCleanup; @@ -143,35 +164,71 @@ export interface HopperApplicationLauncherDependencies { * macOS LaunchServices may create or reuse. */ export class HopperApplicationLauncher implements BridgeLauncher { + readonly #preparedImageResources = new ArtifactResourceScope(); + constructor( readonly options: HopperApplicationLauncherOptions, readonly dependencies: HopperApplicationLauncherDependencies = {}, ) {} async cleanup(): Promise> { - if (!usesLinuxDemo(this.options)) return ok(null); + const failures: ProviderCleanupError[] = []; try { - await ( - this.dependencies.retryPublisherCleanup ?? - retryLinuxHopperPublisherCleanup - )(); - return ok(null); + await this.#preparedImageResources.retryCleanup(); } catch (cause: unknown) { const observation = ArtifactReaderFailure.cleanupObservation( cause, - "hopper-publisher-verification", + this.#preparedImageResources.cleanupResources, ); - return err( - new ProviderCleanupError( + const failure = new ProviderCleanupError( + "hopper", + observation.resources, + { reason: observation.reason }, + { cause }, + ); + bindRetainedArtifactCleanup(failure, cause); + failures.push(failure); + } + if (usesLinuxDemo(this.options)) { + try { + await ( + this.dependencies.retryPublisherCleanup ?? + retryLinuxHopperPublisherCleanup + )(); + } catch (cause: unknown) { + const observation = ArtifactReaderFailure.cleanupObservation( + cause, + "hopper-publisher-verification", + ); + const failure = new ProviderCleanupError( "hopper", observation.resources, - { - reason: observation.reason, - }, + { reason: observation.reason }, { cause }, - ), - ); + ); + bindRetainedArtifactCleanup(failure, cause); + failures.push(failure); + } } + if (failures.length === 0) return ok(null); + if (failures.length === 1) return err(failures[0]!); + const resources = [ + ...new Set(failures.flatMap((failure) => failure.cleanupResources)), + ]; + const failure = new ProviderCleanupError( + "hopper", + resources, + { + reason: failures.map((failure) => failure.message).join("; "), + }, + { + cause: new AggregateError( + failures, + "Hopper image and publisher cleanup remain incomplete", + ), + }, + ); + return err(failure.bindCleanupDependencies(failures)); } async launch( @@ -189,14 +246,39 @@ export class HopperApplicationLauncher implements BridgeLauncher { let leaseTransferred = false; try { let loadedPath = this.options.targetPath; - if (this.options.preparedImage !== undefined) { - const prepared = await prepareHopperMachOImage( - this.options.targetPath, - this.options.preparedImage.sourceSha256, - this.options.preparedImage.image, - join(session.directory, "image.macho"), - options.signal, - ); + const preparedImage = this.options.preparedImage; + if (preparedImage !== undefined) { + let prepared: Awaited>; + try { + prepared = await this.#preparedImageResources.run(() => + prepareHopperMachOImage( + this.options.targetPath, + preparedImage.sourceSha256, + preparedImage.image, + join(session.directory, "image.macho"), + { + ...(options.signal === undefined + ? {} + : { signal: options.signal }), + resourceScope: this.#preparedImageResources, + ...(this.dependencies.imageFileSystem === undefined + ? {} + : { fileSystem: this.dependencies.imageFileSystem }), + }, + ), + ); + } catch (cause: unknown) { + const failure = new HopperStartError({ + cause, + userMessage: `Previous Hopper image cleanup remains incomplete: ${cause instanceof Error ? cause.message : String(cause)}`, + cleanup: { + reason: cause instanceof Error ? cause.message : String(cause), + resources: this.#preparedImageResources.cleanupResources, + }, + }); + bindRetainedArtifactCleanup(failure, cause); + return err(failure); + } if (!prepared.ok) return prepared; loadedPath = prepared.value; } diff --git a/src/hopper/HopperAnalysisProfile.test.ts b/src/hopper/HopperAnalysisProfile.test.ts index e5b5745049..247ac99d0d 100644 --- a/src/hopper/HopperAnalysisProfile.test.ts +++ b/src/hopper/HopperAnalysisProfile.test.ts @@ -1,15 +1,33 @@ import { createHash } from "node:crypto"; +import { PassThrough } from "node:stream"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { afterEach, expect, it, vi } from "vitest"; +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createReadStream: vi.fn(actual.createReadStream), + }; +}); + +import { createReadStream } from "node:fs"; + +import { createInspectionResources } from "../../tests/fixtures/inspectionResources.js"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { hopperLoaderArgsForTarget, resolveHopperAnalysisProfile, } from "./HopperAnalysisProfile.js"; +let inspectionResources = createInspectionResources(); +afterEach(async () => { + await inspectionResources.close(); + inspectionResources = createInspectionResources(); +}); + it("routes physical universal Mach-O containers through Hopper's FAT loader", () => { const result = hopperLoaderArgsForTarget( { @@ -72,6 +90,7 @@ it("prepares a verified thin arm64e slice for a FAT32 system container", async ( launcherPath: path, loaderArgsOverride: [], provider: { id: "hopper", name: "Hopper Disassembler", version: null }, + resourceScope: inspectionResources.owner, }, ); if (!result.ok) throw new Error(result.error.message); @@ -85,9 +104,227 @@ it("prepares a verified thin arm64e slice for a FAT32 system container", async ( cpu_type: 0x0100000c, cpu_subtype: 0x80000002, }, + target_selection: { + kind: "macho-selected-image-v1", + container: "fat", + offset: arm64eOffset, + byte_length: 64, + cpu_type: 0x0100000c, + cpu_subtype: 0x80000002, + container_table_sha256: expect.any(String), + }, loader: { source: "derived", arguments: ["-l", "Mach-O", "--aarch64"], }, }); }); + +it("commits a thin header selection only for derived Hopper loader arguments", async () => { + const directory = await createTestTempDirectory("rea-hopper-thin-profile-"); + const thin = Buffer.alloc(64); + thin.writeUInt32LE(0xfeedfacf, 0); + thin.writeUInt32LE(0x0100000c, 4); + thin.writeUInt32LE(0x80000002, 8); + const path = join(directory, "arm64e.macho"); + await writeFile(path, thin); + const target = { + path, + sha256: createHash("sha256").update(thin).digest("hex"), + kind: "executable" as const, + format: "mach-o" as const, + architecture: "arm64" as const, + availableArchitectures: ["arm64" as const], + }; + const provider = { + id: "hopper", + name: "Hopper Disassembler", + version: null, + }; + const resolved = await resolveHopperAnalysisProfile(target, { + launcherPath: path, + loaderArgsOverride: [], + provider, + resourceScope: inspectionResources.owner, + }); + if (!resolved.ok) throw new Error(resolved.error.message); + expect(resolved.value.profile?.parameters.target_selection).toEqual({ + kind: "macho-selected-image-v1", + container: "thin", + offset: 0, + byte_length: 64, + cpu_type: 0x0100000c, + cpu_subtype: 0x80000002, + container_table_sha256: null, + }); + expect(resolved.value.profile?.parameters.prepared_image).toEqual({ + method: "thin_verified_copy", + byte_length: thin.length, + cpu_type: 0x0100000c, + cpu_subtype: 0x80000002, + }); + + const overridden = await resolveHopperAnalysisProfile(target, { + launcherPath: path, + loaderArgsOverride: ["-l", "Mach-O"], + provider, + resourceScope: inspectionResources.owner, + }); + if (!overridden.ok) throw new Error(overridden.error.message); + expect(overridden.value.profile?.parameters).not.toHaveProperty( + "target_selection", + ); + + const changedSameSize = Buffer.from(thin); + changedSameSize.writeUInt8( + changedSameSize.readUInt8(changedSameSize.length - 1) ^ 1, + changedSameSize.length - 1, + ); + await writeFile(path, changedSameSize); + const staleAdmission = await resolveHopperAnalysisProfile(target, { + launcherPath: path, + loaderArgsOverride: [], + provider, + resourceScope: inspectionResources.owner, + }); + expect(staleAdmission.ok).toBe(false); + if (staleAdmission.ok) throw new Error("stale thin admission was accepted"); + expect(staleAdmission.error.message).toContain( + "Thin Mach-O source changed after target admission", + ); +}); + +it("keeps a read failure primary when cancellation arrives with the stream error", async () => { + const path = "/fixture/arm64.macho"; + const target = { + path, + sha256: "a".repeat(64), + kind: "executable" as const, + format: "mach-o" as const, + architecture: "arm64" as const, + availableArchitectures: ["arm64" as const], + }; + const controller = new AbortController(); + const reason = { request: "cancel-after-read-error" }; + const physicalFailure = new Error("fixture read failure"); + const stream = new PassThrough(); + vi.mocked(createReadStream).mockReturnValueOnce( + stream as unknown as ReturnType, + ); + stream.once("error", () => controller.abort(reason)); + + const resolving = resolveHopperAnalysisProfile(target, { + launcherPath: "/fixture/hopper", + loaderArgsOverride: [], + provider: { id: "hopper", name: "Hopper", version: null }, + resourceScope: inspectionResources.owner, + signal: controller.signal, + }); + stream.destroy(physicalFailure); + + const result = await resolving; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("A failed header read was accepted"); + expect(result.error._tag).toBe("BinaryTargetError"); + expect(result.error.cause).toBe(physicalFailure); +}); + +it("retains the exact caller reason when the owned header stream is aborted", async () => { + const controller = new AbortController(); + const reason = { request: "cancel-header-read" }; + const stream = new PassThrough(); + vi.mocked(createReadStream).mockReturnValueOnce( + stream as unknown as ReturnType, + ); + const resolving = resolveHopperAnalysisProfile( + { + path: "/fixture/arm64.macho", + sha256: "a".repeat(64), + kind: "executable", + format: "mach-o", + architecture: "arm64", + availableArchitectures: ["arm64"], + }, + { + launcherPath: "/fixture/hopper", + loaderArgsOverride: [], + provider: { id: "hopper", name: "Hopper", version: null }, + resourceScope: inspectionResources.owner, + signal: controller.signal, + }, + ); + controller.abort(reason); + + const result = await resolving; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("An aborted header read was accepted"); + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(reason); +}); + +it("keeps failed Mach-O preflight cleanup attached to the supplied profile owner", async () => { + const directory = await createTestTempDirectory("rea-hopper-profile-owner-"); + const sourcePath = join(directory, "source.macho"); + const bytes = Buffer.alloc(100_000, 0x5a); + bytes.writeUInt32BE(0xfeedfacf, 0); + bytes.writeUInt32BE(0x0100000c, 4); + bytes.writeUInt32BE(0x80000002, 8); + await writeFile(sourcePath, bytes); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const actualNodeFs = + await vi.importActual("node:fs"); + vi.mocked(createReadStream).mockReset(); + vi.mocked(createReadStream).mockImplementation(actualNodeFs.createReadStream); + const closeFailure = new Error("profile preflight close failure"); + let allowClose = false; + const imageFileSystem = { + open: async (...args: Parameters) => { + const handle = await actualFs.open(...args); + const closeHandle = handle.close.bind(handle); + Object.defineProperty(handle, "close", { + configurable: true, + value: async (): Promise => { + if (!allowClose) throw closeFailure; + await closeHandle(); + }, + }); + return handle; + }, + }; + const owner = inspectionResources; + + const resolved = await resolveHopperAnalysisProfile( + { + path: sourcePath, + sha256, + kind: "executable", + format: "mach-o", + architecture: "arm64", + availableArchitectures: ["arm64"], + }, + { + launcherPath: sourcePath, + loaderArgsOverride: [], + provider: { id: "hopper", name: "Hopper", version: null }, + resourceScope: owner.owner, + imageFileSystem, + }, + ); + + expect(resolved.ok).toBe(false); + if (resolved.ok) throw new Error("Failed preflight cleanup was accepted"); + expect(resolved.error._tag).toBe("BinaryTargetError"); + expect(resolved.error.cause).toBe(closeFailure); + expect(resolved.error.cleanupIncomplete).toBe(true); + expect(resolved.error.cleanupResources).toEqual([ + `hopper-macho-preflight:${sourcePath}`, + ]); + + allowClose = true; + await owner.close(); + expect(resolved.error.cleanupIncomplete).toBe(false); +}); diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index 26fb390f50..d94d5d9cd8 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -3,6 +3,7 @@ import { createReadStream } from "node:fs"; import type { AnalysisProfileResolution, + RetainedResourceScope, ProviderIdentity, } from "../application/AnalysisProvider.js"; import { createAnalysisProfile } from "../domain/analysisProfile.js"; @@ -19,11 +20,16 @@ import { } from "../domain/analysisErrorCore.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + machoTargetSelectionSchema, + type MachoTargetSelection, +} from "../domain/apple/machoTargetSelection.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; import { resolveHopperMachOImage, + type HopperMachOImageResolutionFileSystem, type HopperMachOImage, } from "./HopperMachOImage.js"; @@ -31,6 +37,8 @@ interface HopperProfileOptions { readonly launcherPath: string; readonly loaderArgsOverride: readonly string[]; readonly provider: ProviderIdentity; + readonly resourceScope: RetainedResourceScope; + readonly imageFileSystem?: HopperMachOImageResolutionFileSystem; readonly signal?: AbortSignal; } @@ -38,6 +46,25 @@ interface HopperProfileOptions { export const resolveHopperAnalysisProfile = async ( target: BinaryTarget, options: HopperProfileOptions, +): Promise> => { + try { + return await options.resourceScope.run(() => + resolveHopperAnalysisProfileInScope(target, options), + ); + } catch (cause: unknown) { + return err( + cause instanceof AnalysisError + ? cause + : new ProviderAdapterError("hopper", "resolve_analysis_profile", { + cause, + }), + ); + } +}; + +const resolveHopperAnalysisProfileInScope = async ( + target: BinaryTarget, + options: HopperProfileOptions, ): Promise> => { if (target.kind !== "executable" && target.kind !== "database") return ok({ profile: null }); @@ -54,13 +81,41 @@ export const resolveHopperAnalysisProfile = async ( let preparedImage: HopperMachOImage | undefined; if ( target.kind === "executable" && - (machoContainer === "fat64" || machoContainer === "fat32") && + machoContainer !== undefined && options.loaderArgsOverride.length === 0 ) { - const resolved = await resolveHopperMachOImage(target, options.signal); + const resolved = await resolveHopperMachOImage(target, { + resourceScope: options.resourceScope, + ...(options.signal === undefined ? {} : { signal: options.signal }), + ...(options.imageFileSystem === undefined + ? {} + : { fileSystem: options.imageFileSystem }), + }); if (!resolved.ok) return resolved; preparedImage = resolved.value; } + let targetSelection: MachoTargetSelection | undefined; + if (preparedImage?.method === "thin_verified_copy") { + targetSelection = machoTargetSelectionSchema.parse({ + kind: "macho-selected-image-v1", + container: "thin", + offset: 0, + byte_length: preparedImage.byte_length, + cpu_type: preparedImage.cpu_type, + cpu_subtype: preparedImage.cpu_subtype, + container_table_sha256: null, + }); + } else if (preparedImage !== undefined) { + targetSelection = machoTargetSelectionSchema.parse({ + kind: "macho-selected-image-v1", + container: "fat", + offset: preparedImage.source_offset, + byte_length: preparedImage.byte_length, + cpu_type: preparedImage.cpu_type, + cpu_subtype: preparedImage.cpu_subtype, + container_table_sha256: preparedImage.container_table_sha256, + }); + } const derived = hopperLoaderArgsForTarget( target, preparedImage === undefined ? machoContainer : "thin", @@ -93,6 +148,9 @@ export const resolveHopperAnalysisProfile = async ( ...(preparedImage === undefined ? {} : { prepared_image: { ...preparedImage } }), + ...(targetSelection === undefined + ? {} + : { target_selection: targetSelection }), loader: { source: options.loaderArgsOverride.length === 0 @@ -189,17 +247,23 @@ const readMachoMagic = async ( signal?: AbortSignal, ): Promise> => { if (signalIsAborted(signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { cause: signal?.reason }), + ); const stream = createReadStream(path, { start: 0, end: 3 }); + const abortError = new Error("Caller cancelled Mach-O header read"); const onAbort = (): void => { - stream.destroy(); + stream.destroy(abortError); }; signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); try { const chunks: Buffer[] = []; for await (const chunk of stream) chunks.push(Buffer.from(chunk)); if (signalIsAborted(signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { cause: signal?.reason }), + ); const bytes = Buffer.concat(chunks); return bytes.length === 4 ? ok(bytes.readUInt32BE(0)) @@ -210,8 +274,10 @@ const readMachoMagic = async ( ), ); } catch (cause: unknown) { - return signalIsAborted(signal) - ? err(new AnalysisCancelledError("open_binary")) + return cause === abortError + ? err( + new AnalysisCancelledError("open_binary", { cause: signal?.reason }), + ) : err( new BinaryTargetError( path, @@ -229,23 +295,31 @@ const sha256File = async ( signal: AbortSignal | undefined, ): Promise> => { if (signalIsAborted(signal)) - return err(new AnalysisCancelledError("open_binary")); + return err( + new AnalysisCancelledError("open_binary", { cause: signal?.reason }), + ); const stream = createReadStream(path); + const abortError = new Error("Caller cancelled launcher fingerprint read"); const onAbort = (): void => { - stream.destroy(); + stream.destroy(abortError); }; signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); try { const hash = createHash("sha256"); for await (const chunk of stream) hash.update(chunk); return signalIsAborted(signal) - ? err(new AnalysisCancelledError("open_binary")) + ? err( + new AnalysisCancelledError("open_binary", { cause: signal?.reason }), + ) : ok(hash.digest("hex")); } catch (cause: unknown) { - // best-effort cleanup: unreadable targets mean no digest; abort still cancels. - void cause; - return signalIsAborted(signal) - ? err(new AnalysisCancelledError("open_binary")) + // Physical fingerprint failures are best-effort; only our own abort + // sentinel turns this optional observation into caller cancellation. + return cause === abortError + ? err( + new AnalysisCancelledError("open_binary", { cause: signal?.reason }), + ) : ok(undefined); } finally { signal?.removeEventListener("abort", onAbort); diff --git a/src/hopper/HopperMachOImage.copyLifecycle.test.ts b/src/hopper/HopperMachOImage.copyLifecycle.test.ts new file mode 100644 index 0000000000..2da4c55d48 --- /dev/null +++ b/src/hopper/HopperMachOImage.copyLifecycle.test.ts @@ -0,0 +1,342 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, expect, it, vi } from "vitest"; + +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { createInspectionResources } from "../../tests/fixtures/inspectionResources.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { BinaryTargetError } from "../domain/configurationErrors.js"; +import { + HopperCancelledError, + HopperStartError, +} from "../domain/hopperErrors.js"; +import { + prepareHopperMachOImage, + resolveHopperMachOImage as resolveHopperMachOImageWithOwner, +} from "./HopperMachOImage.js"; + +let inspectionResources = createInspectionResources(); +afterEach(async () => { + await inspectionResources.close(); + inspectionResources = createInspectionResources(); +}); + +const resolveHopperMachOImage = ( + target: Parameters[0], +) => + resolveHopperMachOImageWithOwner(target, { + resourceScope: inspectionResources.owner, + }); + +type NativeFs = typeof import("node:fs/promises"); +type NativeFilePath = Parameters[0]; + +const createFileSystemWithCloseHook = async ( + onClose: (path: NativeFilePath, close: () => Promise) => Promise, + onRead?: () => Promise, +) => { + const actualFs = await vi.importActual("node:fs/promises"); + return { + open: async (...args: Parameters) => { + const handle = await actualFs.open(...args); + const path = args[0]; + const closeHandle = handle.close.bind(handle); + Object.defineProperty(handle, "close", { + configurable: true, + value: () => onClose(path, closeHandle), + }); + if (onRead !== undefined) { + Object.defineProperty(handle, "read", { + configurable: true, + value: onRead, + }); + } + return handle; + }, + }; +}; + +const prepareWithChunkWriteFailure = ( + fixture: Awaited>, + cause: Error, + options: Omit[4], "observer">, +) => + prepareHopperMachOImage( + fixture.sourcePath, + fixture.sha256, + fixture.image, + fixture.destination, + { + ...options, + observer: { + afterChunkWritten: async () => { + throw cause; + }, + }, + }, + ); + +it("keeps an observer failure primary and closes real source handles", async () => { + const fixture = await createThinCopyFixture("rea-hopper-copy-primary-"); + const writeFailure = new Error("controlled copy observer failure"); + const prepared = await prepareWithChunkWriteFailure(fixture, writeFailure, { + resourceScope: new ArtifactResourceScope(), + }); + + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("A failed copy was accepted"); + expect(prepared.error.cause).toBe(writeFailure); + expect(prepared.error.cleanupIncomplete).toBe(false); +}); + +it("keeps a copy failure primary while retaining and retrying a real source handle", async () => { + const fixture = await createThinCopyFixture("rea-hopper-copy-cleanup-"); + + const firstCloseFailure = new Error("source close is not confirmed"); + let allowSourceClose = false; + let sourceCloseAttempts = 0; + let outputCloseAttempts = 0; + const fileSystem = await createFileSystemWithCloseHook( + async (path, close) => { + if (path === fixture.sourcePath) { + sourceCloseAttempts += 1; + if (!allowSourceClose) throw firstCloseFailure; + } else { + outputCloseAttempts += 1; + } + await close(); + }, + ); + const resourceScope = new ArtifactResourceScope(); + const writeFailure = new Error("controlled copy observer failure"); + const prepared = await prepareWithChunkWriteFailure(fixture, writeFailure, { + resourceScope, + fileSystem, + }); + + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("A failed copy was accepted"); + expect(prepared.error.cause).toBe(writeFailure); + expect(prepared.error.cleanupIncomplete).toBe(true); + expect(prepared.error.cleanupResources).toEqual([ + `hopper-image-source:${fixture.sourcePath}`, + ]); + expect(outputCloseAttempts).toBe(1); + expect(sourceCloseAttempts).toBe(2); + + allowSourceClose = true; + await resourceScope.retryCleanup(); + expect(sourceCloseAttempts).toBe(3); + expect(prepared.error.cause).toBe(writeFailure); + expect(prepared.error.cleanupIncomplete).toBe(false); +}); + +it("keeps an output-close failure primary when cancellation first arrives during source close", async () => { + const fixture = await createThinCopyFixture("rea-hopper-late-close-abort-"); + const controller = new AbortController(); + const reason = { request: "abort-during-source-close" }; + const outputCloseFailure = new Error("destination close failed first"); + let allowOutputClose = false; + const fileSystem = await createFileSystemWithCloseHook( + async (path, close) => { + if (path === fixture.destination && !allowOutputClose) + throw outputCloseFailure; + await close(); + if (path === fixture.sourcePath) controller.abort(reason); + }, + ); + const resourceScope = new ArtifactResourceScope(); + + const prepared = await prepareHopperMachOImage( + fixture.sourcePath, + fixture.sha256, + fixture.image, + fixture.destination, + { fileSystem, resourceScope, signal: controller.signal }, + ); + + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("Failed output closure was accepted"); + expect(prepared.error).toBeInstanceOf(HopperStartError); + expect(prepared.error.cause).toBe(outputCloseFailure); + expect(prepared.error.cause).not.toBe(reason); + expect(prepared.error.cleanupIncomplete).toBe(true); + expect(prepared.error.cleanupResources).toEqual([ + `hopper-prepared-image:${fixture.destination}`, + ]); + + allowOutputClose = true; + await resourceScope.retryCleanup(); + expect(prepared.error.cleanupIncomplete).toBe(false); +}); + +it("keeps cancellation observed before close primary and retains an output-close failure", async () => { + const fixture = await createThinCopyFixture("rea-hopper-cancel-close-error-"); + const controller = new AbortController(); + const reason = { request: "abort-before-close" }; + const outputCloseFailure = new Error("destination close failed"); + let allowOutputClose = false; + const fileSystem = await createFileSystemWithCloseHook( + async (path, close) => { + if (path === fixture.destination && !allowOutputClose) + throw outputCloseFailure; + await close(); + }, + ); + const resourceScope = new ArtifactResourceScope(); + + const prepared = await prepareHopperMachOImage( + fixture.sourcePath, + fixture.sha256, + fixture.image, + fixture.destination, + { + fileSystem, + resourceScope, + signal: controller.signal, + observer: { + afterChunkWritten: async () => controller.abort(reason), + }, + }, + ); + + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("Cancelled copy was accepted"); + expect(prepared.error).toBeInstanceOf(HopperCancelledError); + expect(prepared.error.cause).toBe(reason); + expect(prepared.error.cleanupIncomplete).toBe(true); + expect(prepared.error.cleanupResources).toEqual([ + `hopper-prepared-image:${fixture.destination}`, + ]); + expect( + prepared.error.secondaryFailures.some( + (failure) => failure.cause === outputCloseFailure, + ), + ).toBe(true); + + allowOutputClose = true; + await resourceScope.retryCleanup(); + expect(prepared.error.cleanupIncomplete).toBe(false); +}); + +it("preserves a physical preflight read failure and retains the handle after close fails", async () => { + const fixture = await createThinCopyFixture( + "rea-hopper-preflight-read-close-", + ); + const readFailure = new Error("controlled preflight read failure"); + const closeFailure = new Error("controlled preflight close failure"); + let allowClose = false; + let closeAttempts = 0; + const fileSystem = await createFileSystemWithCloseHook( + async (_path, close) => { + closeAttempts += 1; + if (!allowClose) throw closeFailure; + await close(); + }, + async () => { + throw readFailure; + }, + ); + const owner = inspectionResources; + + const result = await resolveHopperMachOImageWithOwner(fixture.target, { + resourceScope: owner.owner, + fileSystem, + }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Read failure was accepted"); + expect(result.error).toBeInstanceOf(BinaryTargetError); + expect(result.error.cause).toBe(readFailure); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toEqual([ + `hopper-macho-preflight:${fixture.sourcePath}`, + ]); + expect(result.error.secondaryFailures).toHaveLength(1); + expect(result.error.secondaryFailures[0]?.cleanupIncomplete).toBe(true); + expect(closeAttempts).toBe(1); + + allowClose = true; + await owner.close(); + expect(closeAttempts).toBe(2); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.secondaryFailures[0]?.cleanupIncomplete).toBe(false); +}); + +it("retains exact cancellation during preflight close without replacing its close failure", async () => { + const fixture = await createThinCopyFixture( + "rea-hopper-preflight-close-cancel-", + ); + const controller = new AbortController(); + const reason = { request: "abort-during-preflight-close" }; + const closeFailure = new Error("controlled preflight close failure"); + let allowClose = false; + let closeAttempts = 0; + const fileSystem = await createFileSystemWithCloseHook( + async (_path, close) => { + closeAttempts += 1; + if (!allowClose) { + controller.abort(reason); + throw closeFailure; + } + await close(); + }, + ); + const owner = inspectionResources; + + const result = await resolveHopperMachOImageWithOwner(fixture.target, { + resourceScope: owner.owner, + fileSystem, + signal: controller.signal, + }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Close failure was accepted"); + expect(result.error).toBeInstanceOf(BinaryTargetError); + expect(result.error.cause).toBe(closeFailure); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.secondaryFailures).toHaveLength(1); + expect(result.error.secondaryFailures[0]).toBeInstanceOf( + AnalysisCancelledError, + ); + expect(result.error.secondaryFailures[0]?.cause).toBe(reason); + + allowClose = true; + await owner.close(); + expect(closeAttempts).toBe(2); + expect(result.error.cleanupIncomplete).toBe(false); +}); + +const createThinCopyFixture = async (prefix: string) => { + const directory = await createTestTempDirectory(prefix); + const bytes = Buffer.alloc(100_000, 0x5a); + bytes.writeUInt32BE(0xfeedfacf, 0); + bytes.writeUInt32BE(0x0100000c, 4); + bytes.writeUInt32BE(0x80000002, 8); + const sourcePath = join(directory, "source.macho"); + const destination = join(directory, "image.macho"); + await writeFile(sourcePath, bytes); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const target = { + path: sourcePath, + sha256, + kind: "executable" as const, + format: "mach-o" as const, + architecture: "arm64" as const, + availableArchitectures: ["arm64"] as const, + }; + const resolved = await resolveHopperMachOImage({ + ...target, + }); + if (!resolved.ok) throw resolved.error; + return { + sourcePath, + destination, + sha256, + target, + image: resolved.value, + }; +}; diff --git a/src/hopper/HopperMachOImage.test.ts b/src/hopper/HopperMachOImage.test.ts index e4f81ec6a2..0e509b5e58 100644 --- a/src/hopper/HopperMachOImage.test.ts +++ b/src/hopper/HopperMachOImage.test.ts @@ -1,15 +1,26 @@ import { createHash } from "node:crypto"; -import { chmod, readFile, writeFile } from "node:fs/promises"; +import { + access, + chmod, + mkdir, + readFile, + readdir, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { afterEach, expect, it, vi } from "vitest"; +import { createInspectionResources } from "../../tests/fixtures/inspectionResources.js"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { BinaryTargetError } from "../domain/configurationErrors.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { prepareHopperMachOImage, - resolveHopperMachOImage, + resolveHopperMachOImage as resolveHopperMachOImageWithOwner, } from "./HopperMachOImage.js"; +import { HopperApplicationLauncher } from "./BridgeLauncher.js"; const CPU_TYPES = { x86_64: 0x01000007, arm64: 0x0100000c } as const; const FAT32_ALIGNMENT = 12; @@ -105,6 +116,19 @@ const requireResolved = ( return result.value; }; +let inspectionResources = createInspectionResources(); +afterEach(async () => { + await inspectionResources.close(); + inspectionResources = createInspectionResources(); +}); + +const resolveHopperMachOImage = ( + target: Parameters[0], +) => + resolveHopperMachOImageWithOwner(target, { + resourceScope: inspectionResources.owner, + }); + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( "reports a FAT64 source that became unreadable as a host read denial", async () => { @@ -152,7 +176,7 @@ it("rejects an ELF target before attempting FAT source preparation", async () => expect(result.error).toMatchObject({ _tag: "BinaryTargetError", path, - reason: "Hopper FAT image preparation requires a Mach-O target", + reason: "Hopper image preparation requires a Mach-O target", }); }); @@ -214,7 +238,7 @@ it("fails closed when capability variants share the preferred arm64e base", asyn const result = await resolveHopperMachOImage( fat32Target(path, fixture, "arm64"), ); - if (result.ok || result.error._tag !== "BinaryTargetError") + if (result.ok || !(result.error instanceof BinaryTargetError)) throw new Error( "Expected ambiguous FAT selection to return a BinaryTargetError", ); @@ -351,6 +375,7 @@ it("copies exactly the selected FAT32 slice bytes and verifies the source", asyn target.sha256, resolved, destination, + { resourceScope: new ArtifactResourceScope() }, ); if (!prepared.ok) throw new Error(prepared.error.userMessage); const arm64eOffset = fixture.sliceOffsets[1]!; @@ -361,6 +386,197 @@ it("copies exactly the selected FAT32 slice bytes and verifies the source", asyn expect(await readFile(destination)).toEqual(copied); }); +it("retains the caller reason when a FAT copy is cancelled after its final write", async () => { + const directory = await createTestTempDirectory("rea-hopper-fat-cancel-"); + const fixture = buildFat32Container([ + { cpuType: CPU_TYPES.arm64, cpuSubtype: ARM64E }, + ]); + const path = await writeFixture(directory, "system.fat", fixture); + const target = fat32Target(path, fixture, "arm64"); + const image = requireResolved(await resolveHopperMachOImage(target)); + const destination = join(directory, "image.macho"); + const controller = new AbortController(); + const reason = { request: "cancel-after-fat-write" }; + const prepared = await prepareHopperMachOImage( + path, + target.sha256, + image, + destination, + { + resourceScope: new ArtifactResourceScope(), + signal: controller.signal, + observer: { + afterChunkWritten: async (copiedBytes) => { + expect(copiedBytes).toBe(FAT32_SLICE_SIZE); + controller.abort(reason); + }, + }, + }, + ); + + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("Cancelled FAT copy was accepted"); + expect(prepared.error._tag).toBe("HopperCancelledError"); + expect(prepared.error.cause).toBe(reason); + // The private session directory remains the owner of partial output. + await expect(readFile(destination)).resolves.toEqual( + fixture.container.subarray( + fixture.sliceOffsets[0], + fixture.sliceOffsets[0]! + FAT32_SLICE_SIZE, + ), + ); +}); + +it("retains a failed prepared-image close through launcher cleanup and blocks another launch", async () => { + const directory = await createTestTempDirectory("rea-hopper-copy-close-"); + const bytes = thinFixture(); + const path = join(directory, "source.macho"); + await writeFile(path, bytes); + const target = thinTarget(path, bytes); + const image = requireResolved(await resolveHopperMachOImage(target)); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const firstCloseFailure = new Error("first close attempt failed"); + const retryCloseFailure = new Error("cleanup retry failed"); + let allowClose = false; + let sourceCloseAttempts = 0; + const imageFileSystem = { + open: async (...arguments_: Parameters) => { + const handle = await actualFs.open(...arguments_); + if (arguments_[0] !== path) return handle; + const sourceClose = async (): Promise => { + sourceCloseAttempts += 1; + if (!allowClose && sourceCloseAttempts === 1) throw firstCloseFailure; + if (!allowClose) throw retryCloseFailure; + await handle.close(); + }; + return new Proxy(handle, { + get(targetHandle, property) { + if (property === "close") return sourceClose; + const value = Reflect.get(targetHandle, property, targetHandle); + return typeof value === "function" ? value.bind(targetHandle) : value; + }, + }); + }, + }; + + const sessionDirectory = join(directory, "session"); + await mkdir(sessionDirectory); + const launcher = new HopperApplicationLauncher( + { + environment: {}, + launcherPath: join(directory, "hopper"), + targetPath: path, + targetKind: "executable", + loaderArgs: ["-l", "Mach-O", "--aarch64"], + bridgeScriptPath: join(directory, "bridge.py"), + preparedImage: { image, sourceSha256: target.sha256 }, + launchMode: "native", + }, + { platform: "linux", imageFileSystem }, + ); + const initial = await launcher.launch({ + directory: sessionDirectory, + socketPath: join(sessionDirectory, "bridge.sock"), + token: "fixture-token", + runId: "fixture-run-1", + }); + expect(initial.ok).toBe(false); + if (initial.ok) throw new Error("A copy with failed close was accepted"); + expect(initial.error._tag).toBe("HopperStartError"); + expect(initial.error.cause).toBe(firstCloseFailure); + expect(initial.error.cleanupIncomplete).toBe(true); + expect(initial.error.cleanupResources).toEqual([ + `hopper-image-source:${path}`, + ]); + const blocked = await launcher.launch({ + directory: sessionDirectory, + socketPath: join(sessionDirectory, "bridge.sock"), + token: "fixture-token", + runId: "fixture-run-2", + }); + expect(blocked.ok).toBe(false); + if (blocked.ok) throw new Error("A launch bypassed retained file cleanup"); + expect(blocked.error.cleanupIncomplete).toBe(true); + expect(blocked.error.cleanupResources).toEqual([ + `hopper-image-source:${path}`, + ]); + expect(sourceCloseAttempts).toBe(3); + + const cleanupPending = await launcher.cleanup(); + expect(cleanupPending.ok).toBe(false); + if (cleanupPending.ok) + throw new Error("Cleanup without release was reported successful"); + expect(cleanupPending.error.cleanupIncomplete).toBe(true); + expect(cleanupPending.error.cleanupResources).toEqual([ + `hopper-image-source:${path}`, + ]); + expect(sourceCloseAttempts).toBe(4); + + allowClose = true; + await expect(launcher.cleanup()).resolves.toEqual({ ok: true, value: null }); + expect(initial.error.cleanupIncomplete).toBe(false); + expect(blocked.error.cleanupIncomplete).toBe(false); + expect(cleanupPending.error.cleanupIncomplete).toBe(false); + await expect( + readFile(join(sessionDirectory, "image.macho")), + ).resolves.toEqual(bytes); +}); + +it("observes caller cancellation that arrives during final source-handle cleanup", async () => { + const directory = await createTestTempDirectory( + "rea-hopper-copy-close-cancel-", + ); + const bytes = thinFixture(); + const path = join(directory, "source.macho"); + await writeFile(path, bytes); + const target = thinTarget(path, bytes); + const image = requireResolved(await resolveHopperMachOImage(target)); + const controller = new AbortController(); + const reason = { request: "cancel-during-final-close" }; + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + const fileSystem = { + open: async (...arguments_: Parameters) => { + const handle = await actualFs.open(...arguments_); + if (arguments_[0] !== path) return handle; + const closeHandle = handle.close.bind(handle); + Object.defineProperty(handle, "close", { + configurable: true, + value: async (): Promise => { + controller.abort(reason); + await closeHandle(); + }, + }); + return handle; + }, + }; + + const prepared = await prepareHopperMachOImage( + path, + target.sha256, + image, + join(directory, "image.macho"), + { + resourceScope: new ArtifactResourceScope(), + signal: controller.signal, + fileSystem, + }, + ); + + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("Finalization cancellation was ignored"); + expect(prepared.error._tag).toBe("HopperCancelledError"); + expect(prepared.error.cause).toBe(reason); + await expect(readFile(join(directory, "image.macho"))).resolves.toEqual( + bytes, + ); +}); + it("rejects FAT32 preparation when the source table changed after resolution", async () => { const directory = await createTestTempDirectory("rea-hopper-fat32-race-"); const fixture = buildFat32Container([ @@ -379,8 +595,186 @@ it("rejects FAT32 preparation when the source table changed after resolution", a target.sha256, resolved, destination, + { resourceScope: new ArtifactResourceScope() }, ); if (prepared.ok) throw new Error("A mutated FAT32 source passed verification"); expect(prepared.error.userMessage).toMatch(/no longer agrees/u); }); + +const thinFixture = (): Buffer => { + const image = Buffer.alloc(64, 0x5a); + image.writeUInt32BE(0xfeedfacf, 0); + image.writeUInt32BE(CPU_TYPES.arm64, 4); + image.writeUInt32BE(ARM64E_WITH_CAPABILITIES, 8); + return image; +}; + +const thinTarget = (path: string, bytes: Buffer) => ({ + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + kind: "executable" as const, + format: "mach-o" as const, + architecture: "arm64" as const, + availableArchitectures: ["arm64"] as const, +}); + +it("copies an admitted thin Mach-O through the owned verified-image path", async () => { + const directory = await createTestTempDirectory("rea-hopper-thin-copy-"); + const bytes = thinFixture(); + const path = join(directory, "source.macho"); + await writeFile(path, bytes); + const target = thinTarget(path, bytes); + const image = requireResolved(await resolveHopperMachOImage(target)); + expect(image).toEqual({ + method: "thin_verified_copy", + byte_length: bytes.length, + cpu_type: CPU_TYPES.arm64, + cpu_subtype: ARM64E_WITH_CAPABILITIES, + }); + + const destination = join(directory, "private", "image.macho"); + await mkdir(join(directory, "private")); + const prepared = await prepareHopperMachOImage( + path, + target.sha256, + image, + destination, + { resourceScope: new ArtifactResourceScope() }, + ); + if (!prepared.ok) throw new Error(prepared.error.userMessage); + expect(prepared.value).toBe(destination); + expect(await readFile(destination)).toEqual(bytes); + expect(target.path).toBe(path); +}); + +it("does not launch a thin selection when its admitted source changes before copy", async () => { + const directory = await createTestTempDirectory("rea-hopper-thin-race-"); + const bytes = thinFixture(); + const path = join(directory, "source.macho"); + await writeFile(path, bytes); + const target = thinTarget(path, bytes); + const image = requireResolved(await resolveHopperMachOImage(target)); + const changed = Buffer.from(bytes); + changed.writeUInt8( + changed.readUInt8(changed.length - 1) ^ 1, + changed.length - 1, + ); + await writeFile(path, changed); + + const sessionDirectory = join(directory, "session"); + await mkdir(sessionDirectory); + const launcher = new HopperApplicationLauncher( + { + environment: process.env, + launcherPath: join(directory, "hopper"), + targetPath: path, + targetKind: "executable", + loaderArgs: ["-l", "Mach-O", "--aarch64"], + bridgeScriptPath: join(directory, "bridge.py"), + preparedImage: { image, sourceSha256: target.sha256 }, + launchMode: "native", + }, + { platform: "linux" }, + ); + const launched = await launcher.launch({ + directory: sessionDirectory, + socketPath: join(sessionDirectory, "bridge.sock"), + token: "fixture-token", + runId: "fixture-run", + }); + expect(launched.ok).toBe(false); + if (launched.ok) throw new Error("Changed source reached Hopper launch"); + expect(launched.error.userMessage).toContain( + "no longer agrees with its admitted digest, size, or selected header", + ); + await expect( + access(join(sessionDirectory, "bootstrap.py")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + // The session directory remains the cleanup owner for partial private output. + await expect(readdir(sessionDirectory)).resolves.toContain("image.macho"); +}); + +it("keeps caller cancellation primary after verified thin-image preparation", async () => { + const directory = await createTestTempDirectory("rea-hopper-thin-cancel-"); + const bytes = thinFixture(); + const path = join(directory, "source.macho"); + await writeFile(path, bytes); + const target = thinTarget(path, bytes); + const image = requireResolved(await resolveHopperMachOImage(target)); + const controller = new AbortController(); + controller.abort(new Error("caller cancellation")); + const prepared = await prepareHopperMachOImage( + path, + target.sha256, + image, + join(directory, "image.macho"), + { resourceScope: new ArtifactResourceScope(), signal: controller.signal }, + ); + expect(prepared).toMatchObject({ + ok: false, + error: { _tag: "HopperCancelledError" }, + }); +}); + +it("retains an owned partial thin image and exact cancellation reason after a write", async () => { + const directory = await createTestTempDirectory("rea-hopper-thin-partial-"); + const bytes = Buffer.alloc(2 * 1024 * 1024, 0x5a); + bytes.writeUInt32BE(0xfeedfacf, 0); + bytes.writeUInt32BE(CPU_TYPES.arm64, 4); + bytes.writeUInt32BE(ARM64E_WITH_CAPABILITIES, 8); + const path = join(directory, "source.macho"); + await writeFile(path, bytes); + const target = thinTarget(path, bytes); + const image = requireResolved(await resolveHopperMachOImage(target)); + const sessionDirectory = join(directory, "session"); + await mkdir(sessionDirectory); + const destination = join(sessionDirectory, "image.macho"); + const controller = new AbortController(); + const reason = { owner: "caller", request: 44 }; + let releaseWrite: (() => void) | undefined; + let announceWrite: ((copiedBytes: number) => void) | undefined; + const writeGate = new Promise((resolve) => { + releaseWrite = resolve; + }); + const firstWrite = new Promise((resolve) => { + announceWrite = resolve; + }); + const preparing = prepareHopperMachOImage( + path, + target.sha256, + image, + destination, + { + resourceScope: new ArtifactResourceScope(), + signal: controller.signal, + observer: { + afterChunkWritten: async (copiedBytes) => { + announceWrite?.(copiedBytes); + await writeGate; + }, + }, + }, + ); + + const copiedBytes = await firstWrite; + const partial = await readFile(destination); + expect(partial.byteLength).toBe(copiedBytes); + expect(copiedBytes).toBeGreaterThan(0); + expect(copiedBytes).toBeLessThan(bytes.length); + controller.abort(reason); + const finishWrite = releaseWrite; + if (finishWrite === undefined) + throw new Error("Copy barrier was not created"); + finishWrite(); + + const prepared = await preparing; + expect(prepared.ok).toBe(false); + if (prepared.ok) throw new Error("Cancelled partial copy was accepted"); + expect(prepared.error._tag).toBe("HopperCancelledError"); + expect(prepared.error.cause).toBe(reason); + await expect(readFile(destination)).resolves.toEqual(partial); + await expect(access(destination)).resolves.toBeUndefined(); +}); diff --git a/src/hopper/HopperMachOImage.ts b/src/hopper/HopperMachOImage.ts index 4b0a48cf51..0c746b507c 100644 --- a/src/hopper/HopperMachOImage.ts +++ b/src/hopper/HopperMachOImage.ts @@ -2,9 +2,17 @@ import { createHash } from "node:crypto"; import { open, type FileHandle } from "node:fs/promises"; import { z } from "zod"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { + AnalysisCleanupLease, + AnalysisCleanupLeaseGroup, +} from "../artifacts/AnalysisCleanupLease.js"; +import type { RetainedResourceScope } from "../application/AnalysisProvider.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { readFatSliceDeclarations, validateFatSlice, @@ -17,13 +25,29 @@ import { HopperStartError, } from "../domain/hopperErrors.js"; import { err, ok, type Result } from "../domain/result.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js"; -/** Stable source coordinates committed before preparing an owned thin image. */ -export const hopperMachOImageSchema = z.strictObject({ - method: z.union([ - z.literal("fat32_thin_slice"), - z.literal("fat64_thin_slice"), - ]), +/** Narrow file-opening dependency for the owned prepared-image copy. */ +export interface HopperMachOImageFileSystem { + readonly open: typeof open; +} + +export interface HopperMachOImageCopyDependencies { + /** Persistent owner that can retry any retained descriptor cleanup. */ + readonly resourceScope: ArtifactResourceScope; + readonly fileSystem?: HopperMachOImageFileSystem; +} + +export interface HopperMachOImagePreparationOptions extends HopperMachOImageCopyDependencies { + readonly signal?: AbortSignal; + readonly observer?: HopperMachOImageCopyObserver; +} + +const defaultHopperMachOImageFileSystem: HopperMachOImageFileSystem = { open }; + +const hopperFatMachOImageSchema = z.strictObject({ + method: z.enum(["fat32_thin_slice", "fat64_thin_slice"]), architecture_count: z.number().int().min(1).max(128), container_table_sha256: z.string().regex(/^[a-f0-9]{64}$/u), source_offset: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), @@ -32,15 +56,50 @@ export const hopperMachOImageSchema = z.strictObject({ cpu_subtype: z.number().int().nonnegative(), }); -export type HopperMachOImage = z.infer; +const hopperThinMachOImageSchema = z.strictObject({ + method: z.literal("thin_verified_copy"), + byte_length: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + cpu_type: z.number().int().nonnegative(), + cpu_subtype: z.number().int().nonnegative(), +}); -/** Validate the selected FAT slice without depending on Apple's lipo tool. */ +/** Stable source coordinates committed before preparing an owned thin image. */ +export const hopperMachOImageSchema = z.discriminatedUnion("method", [ + hopperFatMachOImageSchema, + hopperThinMachOImageSchema, +]); + +export type HopperMachOImage = z.infer; +type HopperMachOImageResolutionError = + | BinaryTargetError + | AnalysisCancelledError; + +/** Controlled checkpoint for deterministic interruption of a real copy. */ +export interface HopperMachOImageCopyObserver { + readonly afterChunkWritten?: (copiedBytes: number) => Promise; +} + +/** Filesystem access used by the admitted Mach-O preflight. */ +export interface HopperMachOImageResolutionFileSystem { + readonly open: typeof open; +} + +export interface HopperMachOImageResolutionOptions { + readonly resourceScope: RetainedResourceScope; + readonly signal?: AbortSignal; + readonly fileSystem?: HopperMachOImageResolutionFileSystem; +} + +const defaultHopperMachOImageResolutionFileSystem: HopperMachOImageResolutionFileSystem = + { + open, + }; + +/** Validate the selected Mach-O image without depending on Apple's lipo tool. */ export const resolveHopperMachOImage = async ( target: BinaryTarget & { readonly kind: "executable" }, - signal?: AbortSignal, -): Promise< - Result -> => { + options: HopperMachOImageResolutionOptions, +): Promise> => { const invalid = (reason: string, cause?: unknown) => err( new BinaryTargetError( @@ -49,129 +108,349 @@ export const resolveHopperMachOImage = async ( cause === undefined ? undefined : { cause }, ), ); - if (signal?.aborted) return err(new AnalysisCancelledError("open_binary")); + if (options.signal?.aborted) + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal.reason, + }), + ); if (target.format !== "mach-o" && target.format !== "mach-o-universal") - return invalid("Hopper FAT image preparation requires a Mach-O target"); - let file: FileHandle; + return invalid("Hopper image preparation requires a Mach-O target"); + let opened: FileHandle; try { - file = await open(target.path, "r"); + opened = await ( + options.fileSystem ?? defaultHopperMachOImageResolutionFileSystem + ).open(target.path, "r"); } catch (cause: unknown) { // Keep the OS error so a read-permission denial reports access_denied. return invalid( - `Cannot read FAT source for Hopper image preparation: ${cause instanceof Error ? cause.message : String(cause)}`, + `Cannot read Mach-O source for Hopper image preparation: ${cause instanceof Error ? cause.message : String(cause)}`, cause, ); } + return inspectAndFinalizeHopperMachOImage( + target, + new OwnedFileHandle(opened), + options, + ); +}; + +const inspectAndFinalizeHopperMachOImage = async ( + target: BinaryTarget & { readonly kind: "executable" }, + owner: OwnedFileHandle, + options: HopperMachOImageResolutionOptions, +): Promise> => { + let result: Result; try { - const read = async (offset: number, length: number): Promise => { - const bytes = Buffer.alloc(length); - const { bytesRead } = await file.read(bytes, 0, length, offset); - return bytes.subarray(0, bytesRead); - }; - const header = await read(0, 8); - const magic = header.length === 8 ? header.readUInt32BE(0) : 0; - const isFatMagic = [ - 0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca, - ].includes(magic); - if (!isFatMagic) - return invalid( - "FAT source header changed before Hopper image preparation", - ); - const wide = magic === 0xcafebabf || magic === 0xbfbafeca; - const littleEndian = magic === 0xbebafeca || magic === 0xbfbafeca; - const entrySize = wide ? 32 : 20; - const count = littleEndian - ? header.readUInt32LE(4) - : header.readUInt32BE(4); - const size = (await file.stat()).size; - const tableEnd = 8 + count * entrySize; - if (count === 0 || count > 128 || tableEnd > size) - return invalid( - "FAT architecture table is truncated or exceeds the supported 128 entries", - ); - const table = await read(8, count * entrySize); - const parsed = readFatSliceDeclarations(table, { - count, - wide, - littleEndian, - }); - if (parsed.status === "malformed") return invalid(parsed.reason); - const cpuTypes = { x86: 7, x86_64: 0x1000007, arm: 12, arm64: 0x100000c }; - const matches = []; - const ranges = []; - for (const slice of parsed.slices) { - if (signal?.aborted) - return err(new AnalysisCancelledError("open_binary")); - const rangeError = validateFatSlice(slice, size, tableEnd); - if (rangeError !== null) return invalid(rangeError); - if (wide) { - const reserved = littleEndian - ? table.readUInt32LE(slice.index * 32 + 28) - : table.readUInt32BE(slice.index * 32 + 28); - if (reserved !== 0) - return invalid( - `FAT64 architecture ${slice.index} has a nonzero reserved field`, - ); - } - if (slice.size < 12) - return invalid( - `FAT architecture ${slice.index} has a truncated Mach-O header`, - ); - const embedded = await read(slice.offset, 12); - if (embedded.length !== 12) - return invalid(`FAT architecture ${slice.index} became truncated`); - const thinMagic = embedded.readUInt32BE(0); - if (![0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(thinMagic)) - return invalid( - `FAT architecture ${slice.index} does not contain a Mach-O image`, - ); - const little = thinMagic === 0xcefaedfe || thinMagic === 0xcffaedfe; - const identity = { - cpuType: little ? embedded.readUInt32LE(4) : embedded.readUInt32BE(4), - cpuSubtype: little - ? embedded.readUInt32LE(8) - : embedded.readUInt32BE(8), - }; - const identityError = validateFatSlice(slice, size, tableEnd, identity); - if (identityError !== null) return invalid(identityError); - ranges.push(slice); - if (slice.cpuType === cpuTypes[target.architecture]) matches.push(slice); - } - ranges.sort((a, b) => a.offset - b.offset); - for (let index = 1; index < ranges.length; index++) { - const previous = ranges[index - 1]; - const current = ranges[index]; - if ( - previous !== undefined && - current !== undefined && - current.offset < previous.offset + previous.size - ) - return invalid("FAT slices overlap"); - } - const selected = selectHopperSlice(matches, target.architecture); - if (selected === undefined) - return invalid( - `FAT selection for ${target.architecture} is ${matches.length === 0 ? "missing" : "ambiguous"}; extract the exact architecture into a thin image before opening it`, - ); - return ok({ - method: wide ? "fat64_thin_slice" : "fat32_thin_slice", - architecture_count: count, - container_table_sha256: createHash("sha256") - .update(header) - .update(table) - .digest("hex"), - source_offset: selected.offset, - byte_length: selected.size, - cpu_type: selected.cpuType, - cpu_subtype: selected.cpuSubtype, - }); - } catch (cause: unknown) { - return invalid( - `Cannot validate FAT image: ${cause instanceof Error ? cause.message : String(cause)}`, + result = await inspectHopperMachOImage( + target, + owner.handle, + options.signal, ); - } finally { - await file.close(); + } catch (cause: unknown) { + result = + cause instanceof BinaryTargetError || + cause instanceof AnalysisCancelledError + ? err(cause) + : err( + new BinaryTargetError( + target.path, + `Cannot validate Mach-O image: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ), + ); } + const cancellationBeforeClose = options.signal?.aborted === true; + let closeOutcome: + | { readonly kind: "closed" } + | { readonly kind: "failed"; readonly cause: unknown }; + try { + await owner.close(); + closeOutcome = { kind: "closed" }; + } catch (cause: unknown) { + closeOutcome = { kind: "failed", cause }; + } + if (closeOutcome.kind === "closed") { + if (result.ok && options.signal?.aborted) + return err( + new AnalysisCancelledError("open_binary", { + cause: options.signal.reason, + }), + ); + return result; + } + return retainHopperPreflightCloseFailure({ + target, + owner, + resourceScope: options.resourceScope, + result, + cancellationBeforeClose, + signal: options.signal, + closeFailure: closeOutcome.cause, + }); +}; + +const retainHopperPreflightCloseFailure = (options: { + readonly target: BinaryTarget & { readonly kind: "executable" }; + readonly owner: OwnedFileHandle; + readonly resourceScope: RetainedResourceScope; + readonly result: Result; + readonly cancellationBeforeClose: boolean; + readonly signal: AbortSignal | undefined; + readonly closeFailure: unknown; +}): Result => { + const resource = `hopper-macho-preflight:${options.target.path}`; + const lease = AnalysisCleanupLease.forCloseOwner(options.owner, resource); + options.resourceScope.retain( + { + close: () => lease.release(), + get cleanupResources() { + return lease.cleanupResources; + }, + }, + resource, + ); + const cleanup = { + reason: + options.closeFailure instanceof Error + ? options.closeFailure.message + : String(options.closeFailure), + resources: [resource], + }; + const cancellationIsPrimary = + (!options.result.ok && + options.result.error instanceof AnalysisCancelledError) || + (options.result.ok && options.cancellationBeforeClose); + let failure: AnalysisError; + let closeIsPrimary = false; + if (!options.result.ok) { + failure = + options.result.error instanceof BinaryTargetError + ? new BinaryTargetError( + options.result.error.path, + options.result.error.reason, + { + ...(options.result.error.cause === undefined + ? {} + : { cause: options.result.error.cause }), + ...(options.result.error.constraint === undefined + ? {} + : { constraint: options.result.error.constraint }), + cleanup, + }, + ) + : new AnalysisCancelledError("open_binary", { + cause: options.result.error.cause, + cleanup, + }); + } else if (options.cancellationBeforeClose) { + failure = new AnalysisCancelledError("open_binary", { + cause: options.signal?.reason, + cleanup, + }); + } else { + failure = new BinaryTargetError( + options.target.path, + "Could not confirm closure of the Mach-O preflight handle", + { cause: options.closeFailure, cleanup }, + ); + closeIsPrimary = true; + } + const cleanupError = new ProviderCleanupError( + "hopper", + [resource], + { reason: cleanup.reason }, + { + cause: options.closeFailure, + operation: "resolve_analysis_profile", + }, + ); + if (!closeIsPrimary) { + lease.bindCleanupError(cleanupError); + failure.retainSecondaryFailure(cleanupError); + } + if (options.signal?.aborted && !cancellationIsPrimary) + failure.retainSecondaryFailure( + new AnalysisCancelledError("open_binary", { + cause: options.signal.reason, + }), + ); + new AnalysisCleanupLeaseGroup(failure, [lease]); + return err(failure); +}; + +const inspectHopperMachOImage = async ( + target: BinaryTarget & { readonly kind: "executable" }, + file: FileHandle, + signal: AbortSignal | undefined, +): Promise> => { + const invalid = (reason: string, cause?: unknown) => + err( + new BinaryTargetError( + target.path, + reason, + cause === undefined ? undefined : { cause }, + ), + ); + const read = async (offset: number, length: number): Promise => { + const bytes = Buffer.alloc(length); + const { bytesRead } = await file.read(bytes, 0, length, offset); + return bytes.subarray(0, bytesRead); + }; + const header = await read(0, 8); + const magic = header.length === 8 ? header.readUInt32BE(0) : 0; + if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) { + if (target.format !== "mach-o") + return invalid("Thin Mach-O source disagrees with its admitted format"); + const prefix = Buffer.alloc(12); + const hash = createHash("sha256"); + let prefixLength = 0; + let size = 0; + for await (const chunk of readFileHandleChunks(file, { + ...(signal === undefined ? {} : { signal }), + cancellationFailure: (reason) => + new AnalysisCancelledError("open_binary", { cause: reason }), + })) { + if (signal?.aborted) + return err( + new AnalysisCancelledError("open_binary", { + cause: signal.reason, + }), + ); + const bytes = Buffer.from(chunk); + hash.update(bytes); + size += bytes.length; + if (!Number.isSafeInteger(size)) + return invalid("Thin Mach-O exceeds exact selection coordinates"); + const copied = Math.min(prefix.length - prefixLength, bytes.length); + if (copied > 0) { + bytes.copy(prefix, prefixLength, 0, copied); + prefixLength += copied; + } + } + if (signal?.aborted) + return err( + new AnalysisCancelledError("open_binary", { cause: signal.reason }), + ); + if (prefixLength !== prefix.length) + return invalid("Thin Mach-O header is truncated"); + const observedSha256 = hash.digest("hex"); + if (observedSha256 !== target.sha256) + return invalid( + `Thin Mach-O source changed after target admission: expected SHA-256 ${target.sha256}, observed ${observedSha256}`, + ); + const little = magic === 0xcefaedfe || magic === 0xcffaedfe; + const headerMagic = prefix.readUInt32BE(0); + if (headerMagic !== magic) + return invalid("Thin Mach-O header changed during profile resolution"); + return ok({ + method: "thin_verified_copy", + byte_length: size, + cpu_type: little ? prefix.readUInt32LE(4) : prefix.readUInt32BE(4), + cpu_subtype: little ? prefix.readUInt32LE(8) : prefix.readUInt32BE(8), + }); + } + const isFatMagic = [0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca].includes( + magic, + ); + if (!isFatMagic) + return invalid( + "Mach-O source header changed before Hopper image preparation", + ); + const wide = magic === 0xcafebabf || magic === 0xbfbafeca; + const littleEndian = magic === 0xbebafeca || magic === 0xbfbafeca; + const entrySize = wide ? 32 : 20; + const count = littleEndian ? header.readUInt32LE(4) : header.readUInt32BE(4); + const size = (await file.stat()).size; + const tableEnd = 8 + count * entrySize; + if (count === 0 || count > 128 || tableEnd > size) + return invalid( + "FAT architecture table is truncated or exceeds the supported 128 entries", + ); + const table = await read(8, count * entrySize); + const parsed = readFatSliceDeclarations(table, { + count, + wide, + littleEndian, + }); + if (parsed.status === "malformed") return invalid(parsed.reason); + if ( + target.architecture !== "x86" && + target.architecture !== "x86_64" && + target.architecture !== "arm" && + target.architecture !== "arm64" + ) + return invalid("Hopper cannot select the admitted Mach-O architecture"); + const architecture = target.architecture; + const cpuTypes = { x86: 7, x86_64: 0x1000007, arm: 12, arm64: 0x100000c }; + const matches = []; + const ranges = []; + for (const slice of parsed.slices) { + if (signal?.aborted) + return err( + new AnalysisCancelledError("open_binary", { cause: signal.reason }), + ); + const rangeError = validateFatSlice(slice, size, tableEnd); + if (rangeError !== null) return invalid(rangeError); + if (wide) { + const reserved = littleEndian + ? table.readUInt32LE(slice.index * 32 + 28) + : table.readUInt32BE(slice.index * 32 + 28); + if (reserved !== 0) + return invalid( + `FAT64 architecture ${slice.index} has a nonzero reserved field`, + ); + } + if (slice.size < 12) + return invalid( + `FAT architecture ${slice.index} has a truncated Mach-O header`, + ); + const embedded = await read(slice.offset, 12); + if (embedded.length !== 12) + return invalid(`FAT architecture ${slice.index} became truncated`); + const thinMagic = embedded.readUInt32BE(0); + if (![0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(thinMagic)) + return invalid( + `FAT architecture ${slice.index} does not contain a Mach-O image`, + ); + const little = thinMagic === 0xcefaedfe || thinMagic === 0xcffaedfe; + const identity = { + cpuType: little ? embedded.readUInt32LE(4) : embedded.readUInt32BE(4), + cpuSubtype: little ? embedded.readUInt32LE(8) : embedded.readUInt32BE(8), + }; + const identityError = validateFatSlice(slice, size, tableEnd, identity); + if (identityError !== null) return invalid(identityError); + ranges.push(slice); + if (slice.cpuType === cpuTypes[architecture]) matches.push(slice); + } + ranges.sort((a, b) => a.offset - b.offset); + for (let index = 1; index < ranges.length; index++) { + const previous = ranges[index - 1]; + const current = ranges[index]; + if ( + previous !== undefined && + current !== undefined && + current.offset < previous.offset + previous.size + ) + return invalid("FAT slices overlap"); + } + const selected = selectHopperSlice(matches, architecture); + if (selected === undefined) + return invalid( + `FAT selection for ${architecture} is ${matches.length === 0 ? "missing" : "ambiguous"}; extract the exact architecture into a thin image before opening it`, + ); + return ok({ + method: wide ? "fat64_thin_slice" : "fat32_thin_slice", + architecture_count: count, + container_table_sha256: createHash("sha256") + .update(header) + .update(table) + .digest("hex"), + source_offset: selected.offset, + byte_length: selected.size, + cpu_type: selected.cpuType, + cpu_subtype: selected.cpuSubtype, + }); }; /** @@ -202,87 +481,398 @@ export const prepareHopperMachOImage = async ( sourceSha256: string, image: HopperMachOImage, destination: string, - signal?: AbortSignal, + options: HopperMachOImagePreparationOptions, ): Promise> => { - if (signal?.aborted) return err(new HopperCancelledError()); + const copyDependencies = { + resourceScope: options.resourceScope, + fileSystem: options.fileSystem ?? defaultHopperMachOImageFileSystem, + }; + const { signal, observer } = options; + if (signal?.aborted) return err(new HopperCancelledError(signal.reason)); + if (image.method === "thin_verified_copy") { + const prepared = await prepareThinHopperMachOImage( + sourcePath, + sourceSha256, + image, + destination, + { signal, observer, ...copyDependencies }, + ); + return signal?.aborted && prepared.ok + ? err(new HopperCancelledError(signal.reason)) + : prepared; + } const tableEntrySize = image.method === "fat64_thin_slice" ? 32 : 20; - try { - const source = await open(sourcePath, "r"); - try { - const output = await open(destination, "wx", 0o600); - try { - const hash = createHash("sha256"); - const tableHash = createHash("sha256"); - const tableEnd = 8 + image.architecture_count * tableEntrySize; - const thinHeader: Buffer[] = []; - let position = 0; - let copied = 0; - for await (const chunk of source.createReadStream({ - autoClose: false, - })) { - if (signal?.aborted) return err(new HopperCancelledError()); - const bytes = Buffer.from(chunk); - hash.update(bytes); - if (position < tableEnd) - tableHash.update( - bytes.subarray(0, Math.min(bytes.length, tableEnd - position)), - ); - const start = Math.max(0, image.source_offset - position); - const end = Math.min( - bytes.length, - image.source_offset + image.byte_length - position, - ); - if (end > start) { - if (copied < 12) - thinHeader.push( - bytes.subarray(start, Math.min(end, start + 12 - copied)), - ); - await output.writeFile(bytes.subarray(start, end)); - copied += end - start; - } - position += bytes.length; - } - const observed = hash.digest("hex"); - const header = Buffer.concat(thinHeader); - const magic = header.length === 12 ? header.readUInt32BE(0) : 0; - const little = magic === 0xcefaedfe || magic === 0xcffaedfe; - const headerMatches = - header.length === 12 && - [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic) && - (little ? header.readUInt32LE(4) : header.readUInt32BE(4)) === - image.cpu_type && - (little ? header.readUInt32LE(8) : header.readUInt32BE(8)) === - image.cpu_subtype; - if ( - tableHash.digest("hex") !== image.container_table_sha256 || - !headerMatches - ) - return err( - new HopperStartError({ - userMessage: `FAT source ${sourcePath} no longer agrees with its committed architecture table or selected Mach-O header. Open the stable source again.`, - }), - ); - if (observed !== sourceSha256 || copied !== image.byte_length) - return err( - new HopperStartError({ - userMessage: `FAT source ${sourcePath} changed before launch: expected SHA-256 ${sourceSha256}, observed ${observed}; copied ${copied} of ${image.byte_length} selected bytes. Open the stable source again.`, - }), - ); - return ok(destination); - } finally { - await output.close(); - } - } finally { - await source.close(); - } - } catch (cause: unknown) { - return signal?.aborted - ? err(new HopperCancelledError()) - : err( - new HopperStartError({ - cause, - userMessage: `Cannot prepare the selected FAT image from ${sourcePath}: ${cause instanceof Error ? cause.message : String(cause)}`, - }), + const hash = createHash("sha256"); + const tableHash = createHash("sha256"); + const tableEnd = 8 + image.architecture_count * tableEntrySize; + const thinHeader: Buffer[] = []; + let copied = 0; + const copiedResult = await copyHopperSource({ + sourcePath, + destination, + signal, + observer, + ...copyDependencies, + selectBytes: (bytes, position) => { + hash.update(bytes); + if (position < tableEnd) + tableHash.update( + bytes.subarray(0, Math.min(bytes.length, tableEnd - position)), ); + const start = Math.max(0, image.source_offset - position); + const end = Math.min( + bytes.length, + image.source_offset + image.byte_length - position, + ); + if (end <= start) return undefined; + if (copied < 12) + thinHeader.push( + bytes.subarray(start, Math.min(end, start + 12 - copied)), + ); + copied += end - start; + return bytes.subarray(start, end); + }, + failureContext: `Cannot prepare the selected FAT image from ${sourcePath}`, + }); + if (!copiedResult.ok) return copiedResult; + const observed = hash.digest("hex"); + const header = Buffer.concat(thinHeader); + const magic = header.length === 12 ? header.readUInt32BE(0) : 0; + const little = magic === 0xcefaedfe || magic === 0xcffaedfe; + const headerMatches = + header.length === 12 && + [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic) && + (little ? header.readUInt32LE(4) : header.readUInt32BE(4)) === + image.cpu_type && + (little ? header.readUInt32LE(8) : header.readUInt32BE(8)) === + image.cpu_subtype; + if ( + tableHash.digest("hex") !== image.container_table_sha256 || + !headerMatches + ) + return err( + new HopperStartError({ + userMessage: `FAT source ${sourcePath} no longer agrees with its committed architecture table or selected Mach-O header. Open the stable source again.`, + }), + ); + if (observed !== sourceSha256 || copied !== image.byte_length) + return err( + new HopperStartError({ + userMessage: `FAT source ${sourcePath} changed before launch: expected SHA-256 ${sourceSha256}, observed ${observed}; copied ${copied} of ${image.byte_length} selected bytes. Open the stable source again.`, + }), + ); + return signal?.aborted + ? err(new HopperCancelledError(signal.reason)) + : ok(destination); +}; + +const prepareThinHopperMachOImage = async ( + sourcePath: string, + sourceSha256: string, + image: Extract, + destination: string, + options: { + readonly signal: AbortSignal | undefined; + readonly observer: HopperMachOImageCopyObserver | undefined; + readonly resourceScope: ArtifactResourceScope; + readonly fileSystem: HopperMachOImageFileSystem; + }, +): Promise> => { + const { signal, observer, resourceScope, fileSystem } = options; + const hash = createHash("sha256"); + const prefix = Buffer.alloc(12); + let prefixLength = 0; + const copiedResult = await copyHopperSource({ + sourcePath, + destination, + signal, + observer, + resourceScope, + fileSystem, + selectBytes: (bytes) => { + hash.update(bytes); + const prefixBytes = Math.min(prefix.length - prefixLength, bytes.length); + if (prefixBytes > 0) { + bytes.copy(prefix, prefixLength, 0, prefixBytes); + prefixLength += prefixBytes; + } + return bytes; + }, + failureContext: `Cannot prepare the verified thin image from ${sourcePath}`, + }); + if (!copiedResult.ok) return copiedResult; + const copied = copiedResult.value; + if (!Number.isSafeInteger(copied)) + return err( + new HopperStartError({ + userMessage: `Thin Mach-O source ${sourcePath} exceeds exact copy coordinates.`, + }), + ); + const observed = hash.digest("hex"); + const magic = prefixLength === prefix.length ? prefix.readUInt32BE(0) : 0; + const little = magic === 0xcefaedfe || magic === 0xcffaedfe; + const headerMatches = + prefixLength === prefix.length && + [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic) && + (little ? prefix.readUInt32LE(4) : prefix.readUInt32BE(4)) === + image.cpu_type && + (little ? prefix.readUInt32LE(8) : prefix.readUInt32BE(8)) === + image.cpu_subtype; + if ( + observed !== sourceSha256 || + copied !== image.byte_length || + !headerMatches + ) + return err( + new HopperStartError({ + userMessage: `Thin Mach-O source ${sourcePath} no longer agrees with its admitted digest, size, or selected header. Open the stable source again.`, + }), + ); + return signal?.aborted + ? err(new HopperCancelledError(signal.reason)) + : ok(destination); +}; + +type HopperImageHandle = OwnedFileHandle | undefined; +type HopperImageCloseFailure = { + readonly resource: string; + readonly cause: unknown; +}; +type HopperImageCleanupAttempt = Extract< + Awaited>, + { readonly kind: "failed" } +>; + +interface HopperImageCopyOperation { + readonly copied: number; + readonly operationFailure: + | { readonly failed: false } + | { + readonly failed: true; + readonly cause: unknown; + }; + readonly cancelled: + | { readonly cancelled: false } + | { + readonly cancelled: true; + readonly reason: unknown; + }; + readonly source: HopperImageHandle; + readonly output: HopperImageHandle; +} + +interface HopperImageCopyFinalization { + readonly closeFailures: readonly HopperImageCloseFailure[]; + readonly retained: readonly HopperImageCleanupAttempt[]; +} + +/** Own source/output handles and copy finalization for both Mach-O layouts. */ +const copyHopperSource = async (options: { + readonly sourcePath: string; + readonly destination: string; + readonly signal: AbortSignal | undefined; + readonly observer: HopperMachOImageCopyObserver | undefined; + readonly resourceScope: ArtifactResourceScope; + readonly fileSystem: HopperMachOImageFileSystem; + readonly selectBytes: (chunk: Buffer, position: number) => Buffer | undefined; + readonly failureContext: string; +}): Promise> => { + if (options.signal?.aborted) + return err(new HopperCancelledError(options.signal.reason)); + const operation = await performHopperImageCopy(options); + const cancellationBeforeFinalization = operation.cancelled.cancelled + ? operation.cancelled + : options.signal?.aborted === true + ? { cancelled: true as const, reason: options.signal.reason } + : { cancelled: false as const }; + const finalization = await closeHopperImageHandles(operation, options); + const cancelled = + cancellationBeforeFinalization.cancelled || + (finalization.closeFailures.length === 0 && + options.signal?.aborted === true); + const cancelReason = cancellationBeforeFinalization.cancelled + ? cancellationBeforeFinalization.reason + : options.signal?.reason; + if ( + !operation.operationFailure.failed && + !cancelled && + finalization.closeFailures.length === 0 + ) + return ok(operation.copied); + return hopperImageCopyFailure({ + operation, + finalization, + failureContext: options.failureContext, + cancelled, + cancelReason, + }); +}; + +const performHopperImageCopy = async (options: { + readonly sourcePath: string; + readonly destination: string; + readonly signal: AbortSignal | undefined; + readonly observer: HopperMachOImageCopyObserver | undefined; + readonly fileSystem: HopperMachOImageFileSystem; + readonly selectBytes: (chunk: Buffer, position: number) => Buffer | undefined; +}): Promise => { + let copied = 0; + let source: HopperImageHandle; + let output: HopperImageHandle; + try { + source = new OwnedFileHandle( + await options.fileSystem.open(options.sourcePath, "r"), + ); + output = new OwnedFileHandle( + await options.fileSystem.open(options.destination, "wx", 0o600), + ); + } catch (cause: unknown) { + return { + copied, + operationFailure: { failed: true, cause }, + cancelled: { cancelled: false }, + source, + output, + }; + } + + try { + let position = 0; + for await (const bytes of readFileHandleChunks(source.handle)) { + if (options.signal?.aborted) + return { + copied, + operationFailure: { failed: false }, + cancelled: { cancelled: true, reason: options.signal.reason }, + source, + output, + }; + const selected = options.selectBytes(bytes, position); + if (selected !== undefined && selected.length > 0) { + await output.handle.writeFile(selected); + copied += selected.length; + await options.observer?.afterChunkWritten?.(copied); + } + position += bytes.length; + } + return { + copied, + operationFailure: { failed: false }, + cancelled: { cancelled: false }, + source, + output, + }; + } catch (cause: unknown) { + return { + copied, + operationFailure: { failed: true, cause }, + cancelled: { cancelled: false }, + source, + output, + }; } }; + +const closeHopperImageHandles = async ( + operation: HopperImageCopyOperation, + options: { + readonly sourcePath: string; + readonly destination: string; + readonly resourceScope: ArtifactResourceScope; + }, +): Promise => { + const closeFailures: HopperImageCloseFailure[] = []; + const retained: HopperImageCleanupAttempt[] = []; + const close = async ( + owner: HopperImageHandle, + resource: string, + ): Promise => { + if (owner === undefined) return; + try { + await owner.close(); + } catch (cause: unknown) { + closeFailures.push({ resource, cause }); + const attempt = await options.resourceScope.release({ + kind: "file-handle", + resource, + handle: owner, + }); + if (attempt.kind === "failed") retained.push(attempt); + } + }; + await close(operation.output, `hopper-prepared-image:${options.destination}`); + await close(operation.source, `hopper-image-source:${options.sourcePath}`); + return { closeFailures, retained }; +}; + +const hopperImageCopyFailure = (options: { + readonly operation: HopperImageCopyOperation; + readonly finalization: HopperImageCopyFinalization; + readonly failureContext: string; + readonly cancelled: boolean; + readonly cancelReason: unknown; +}): Result => { + const { operation, finalization } = options; + const cleanupErrors = finalization.retained.map((attempt) => { + const cleanupError = new ProviderCleanupError( + "hopper", + [attempt.owner.resource], + { + reason: + attempt.cause instanceof Error + ? attempt.cause.message + : String(attempt.cause), + }, + { cause: attempt.cause }, + ); + attempt.lease.bindCleanupError(cleanupError); + return cleanupError; + }); + const cleanup = + cleanupErrors.length === 0 + ? undefined + : { + reason: cleanupErrors.map((failure) => failure.message).join("; "), + resources: [ + ...new Set( + cleanupErrors.flatMap((failure) => failure.cleanupResources), + ), + ], + }; + const primaryFailure = operation.operationFailure.failed + ? operation.operationFailure.cause + : finalization.closeFailures[0]?.cause; + const resultError = + options.cancelled && !operation.operationFailure.failed + ? new HopperCancelledError( + options.cancelReason, + cleanup === undefined ? {} : { cleanup }, + ) + : new HopperStartError({ + ...(operation.operationFailure.failed || + finalization.closeFailures.length > 0 + ? { cause: primaryFailure } + : {}), + userMessage: `${options.failureContext}: ${primaryFailure instanceof Error ? primaryFailure.message : String(primaryFailure ?? "caller cancellation")}`, + ...(cleanup === undefined ? {} : { cleanup }), + }); + for (const closeFailure of finalization.closeFailures) { + if ( + !operation.operationFailure.failed && + !options.cancelled && + closeFailure === finalization.closeFailures[0] + ) + continue; + resultError.retainSecondaryFailure( + new HopperStartError({ + cause: closeFailure.cause, + userMessage: `Could not confirm closure of ${closeFailure.resource}.`, + }), + ); + } + for (const cleanupError of cleanupErrors) + resultError.retainSecondaryFailure(cleanupError); + if (cleanupErrors.length > 0) + resultError.bindCleanupDependencies(cleanupErrors); + return err(resultError); +}; diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index 6108944405..c9c9299474 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -24,6 +24,14 @@ import type { JsonValue } from "../domain/jsonValue.js"; import type { Logger } from "pino"; import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; import { err, ok } from "../domain/result.js"; +import type { RetainedResourceScope } from "../application/AnalysisProvider.js"; +import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; +import { AnalysisCleanupLeaseGroup } from "../artifacts/AnalysisCleanupLease.js"; +import { + ArtifactResourceRetryFailure, + ArtifactResourceScope, +} from "../artifacts/ArtifactResourceScope.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { HopperApplicationLauncher } from "./BridgeLauncher.js"; import { hopperDeclinedArchitectureReason, @@ -44,6 +52,16 @@ import { /** Concrete analysis provider backed by REA's private Hopper bridge. */ export class HopperProvider implements AnalysisProviderCandidate { private readonly environment: Readonly; + readonly #ownedResources = new ArtifactResourceScope(); + readonly #inspectionOwner: RetainedResourceScope = { + retain: (owner, resource) => + this.#ownedResources.retain({ + kind: "lease", + resource, + lease: AnalysisCleanupLease.forCloseOwner(owner, resource), + }), + run: (operation) => this.#ownedResources.run(operation), + }; /** * Host platform, injected so availability and launch-mode decisions can be @@ -169,14 +187,37 @@ export class HopperProvider implements AnalysisProviderCandidate { target: BinaryTarget, options?: AnalysisProfileResolutionOptions, ) { + const resourceScope = options?.resourceScope ?? this.#inspectionOwner; return resolveHopperAnalysisProfile(target, { launcherPath: this.config.hopperLauncherPath, loaderArgsOverride: this.config.hopperLoaderArgs, provider: HOPPER_PROVIDER_IDENTITY, + resourceScope, ...(options?.signal === undefined ? {} : { signal: options.signal }), }); } + hasPendingOwnedResources(): boolean { + return this.#ownedResources.hasPending; + } + + async closeOwnedResources() { + try { + await this.#ownedResources.close(); + return ok(null); + } catch (cause: unknown) { + const failure = new ProviderCleanupError( + HOPPER_PROVIDER_IDENTITY.id, + this.#ownedResources.cleanupResources, + { reason: cause instanceof Error ? cause.message : String(cause) }, + { cause, operation: "close_owned_resources" }, + ); + if (cause instanceof ArtifactResourceRetryFailure) + new AnalysisCleanupLeaseGroup(failure, cause.cleanupLeases); + return err(failure); + } + } + createClient( target: BinaryTarget, profile?: AnalysisProfileCommitment, @@ -287,7 +328,7 @@ export class HopperProvider implements AnalysisProviderCandidate { ...(preparedImage === undefined ? [] : [ - "REA loaded a verified thin Mach-O slice from the original FAT source. Identity and file offsets refer to the original file. The owned temporary image and its Hopper document close together, including on MCP shutdown.", + "REA loaded a verified thin Mach-O image prepared from the original source. Identity and file offsets refer to the original file. The owned temporary image and its Hopper document close together, including on MCP shutdown.", ]), ...(CAPABILITIES.find( (descriptor) => descriptor.operation === operation, diff --git a/tests/boundary/filesystem/directAnalysisPermissions.test.ts b/tests/boundary/filesystem/directAnalysisPermissions.test.ts index e9e1838288..20cf0a7b46 100644 --- a/tests/boundary/filesystem/directAnalysisPermissions.test.ts +++ b/tests/boundary/filesystem/directAnalysisPermissions.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { withArtifactResourceScope } from "../../fixtures/artifactInventory.js"; +import { createInspectionResources } from "../../fixtures/inspectionResources.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { writeAnalysisSnapshot } from "../../../src/application/binary/AnalysisSnapshotFiles.js"; @@ -21,9 +22,13 @@ import { workflowAnalysisProfile, } from "../../../src/application/InvestigationProviders.js"; import { HOPPER_PROVIDER_IDENTITY } from "../../../src/hopper/HopperProviderCapabilities.js"; + import { resolveHopperAnalysisProfile } from "../../../src/hopper/HopperAnalysisProfile.js"; +let inspectionResources = createInspectionResources(); afterEach(async () => { + await inspectionResources.close(); + inspectionResources = createInspectionResources(); vi.unstubAllEnvs(); }); @@ -67,6 +72,7 @@ describe("direct analysis snapshot files", () => { launcherPath, loaderArgsOverride: [], provider: HOPPER_PROVIDER_IDENTITY, + resourceScope: inspectionResources.owner, }); if (!resolved.ok || resolved.value.profile === null) throw new Error("fixture Hopper profile did not resolve"); diff --git a/tests/boundary/providers/hopper/bridgeLauncher.test.ts b/tests/boundary/providers/hopper/bridgeLauncher.test.ts index 85a561bfd9..5d2611ecc6 100644 --- a/tests/boundary/providers/hopper/bridgeLauncher.test.ts +++ b/tests/boundary/providers/hopper/bridgeLauncher.test.ts @@ -1,6 +1,6 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; -import { access, chmod, rm, writeFile } from "node:fs/promises"; +import { access, chmod, open, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; @@ -22,6 +22,8 @@ import { err, ok } from "../../../../src/domain/result.js"; import { HopperReleaseError } from "../../../../src/application/HopperReleaseFeed.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourceScope.js"; +import { AnalysisCleanupLease } from "../../../../src/artifacts/AnalysisCleanupLease.js"; +import { ArtifactReaderFailure } from "../../../../src/artifacts/ArtifactReader.js"; import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js"; import { HopperClient } from "../../../../src/hopper/HopperClient.js"; @@ -64,6 +66,140 @@ const options = (launcherPath: string): HopperApplicationLauncherOptions => ({ demoHelperPath: "/rea/hopper-demo-x11.py", }); +it("resolves mixed image and publisher cleanup only after each owner releases", async () => { + const directory = await createTestTempDirectory("rea-hopper-mixed-cleanup-"); + const targetPath = join(directory, "target.macho"); + const source = Buffer.alloc(64); + source.writeUInt32BE(0xfeedfacf, 0); + source.writeUInt32BE(0x0100000c, 4); + source.writeUInt32BE(0x80000002, 8); + await writeFile(targetPath, source); + + let allowImageClose = false; + let imageCloseAttempts = 0; + const imageFileSystem = { + async open( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ) { + const handle = await open(path, flags, mode); + if (String(path) === targetPath && flags === "r") { + const close = handle.close.bind(handle); + handle.close = async () => { + imageCloseAttempts += 1; + if (!allowImageClose) + throw new Error("Fixture image source close unavailable"); + await close(); + }; + } + return handle; + }, + }; + + let allowPublisherClose = false; + let publisherCloseAttempts = 0; + const publisherOwner = { + async close() { + publisherCloseAttempts += 1; + if (!allowPublisherClose) + throw new Error("Fixture publisher close unavailable"); + }, + }; + const publisherLease = AnalysisCleanupLease.forCloseOwner( + publisherOwner, + "hopper-publisher-verification", + ); + const retryPublisherCleanup = async () => { + try { + await publisherLease.release(); + } catch (cause: unknown) { + const failure = new ArtifactReaderFailure( + "unavailable", + "Publisher verification cleanup remains incomplete", + { + cause, + cleanup: { + reason: cause instanceof Error ? cause.message : String(cause), + resources: ["hopper-publisher-verification"], + }, + cleanupLeases: [publisherLease], + }, + ); + throw failure; + } + }; + const launcher = new HopperApplicationLauncher( + { + ...options("/missing/hopper"), + targetPath, + preparedImage: { + image: { + method: "thin_verified_copy", + byte_length: source.length, + cpu_type: 0x0100000c, + cpu_subtype: 0x80000002, + }, + sourceSha256: createHash("sha256").update(source).digest("hex"), + }, + }, + { + platform: "win32", + imageFileSystem, + retryPublisherCleanup, + }, + ); + const session = { + directory, + socketPath: join(directory, "bridge.sock"), + token: "fixture-token", + runId: "fixture-run", + }; + + try { + const launched = await launcher.launch(session); + expect(launched).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + + const first = await launcher.cleanup(); + expect(first).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + if (first.ok) throw new Error("Expected mixed cleanup to remain pending"); + const originalAggregate = first.error; + expect(originalAggregate.cleanupResources).toEqual( + expect.arrayContaining([ + expect.stringContaining("hopper-image-source:"), + "hopper-publisher-verification", + ]), + ); + + allowImageClose = true; + const publisherOnly = await launcher.cleanup(); + expect(publisherOnly).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(originalAggregate.cleanupIncomplete).toBe(true); + const closedImageAttempts = imageCloseAttempts; + expect(closedImageAttempts).toBeGreaterThan(1); + + allowPublisherClose = true; + expect(await launcher.cleanup()).toEqual({ ok: true, value: null }); + expect(originalAggregate.cleanupIncomplete).toBe(false); + expect(imageCloseAttempts).toBe(closedImageAttempts); + expect(publisherCloseAttempts).toBe(3); + } finally { + allowImageClose = true; + allowPublisherClose = true; + await launcher.cleanup(); + await rm(directory, { recursive: true, force: true }); + } +}); + it("retains cancelled verification cleanup through client shutdown and retries it", async () => { const directory = await createTestTempDirectory("rea-publisher-cancel-"); const targetPath = join(directory, "target"); diff --git a/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts b/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts index 46b71deb77..af087e7b1a 100644 --- a/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts +++ b/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { createInspectionResources } from "../../../fixtures/inspectionResources.js"; import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; import { @@ -12,9 +13,12 @@ import { } from "../../../../src/hopper/HopperAnalysisProfile.js"; import { HOPPER_PROVIDER_IDENTITY } from "../../../../src/hopper/HopperProviderCapabilities.js"; +let inspectionResources = createInspectionResources(); let directory: string | undefined; afterEach(async () => { + await inspectionResources.close(); + inspectionResources = createInspectionResources(); if (directory !== undefined) await rm(directory, { recursive: true, force: true }); directory = undefined; @@ -70,6 +74,7 @@ describe("Hopper analysis profiles", () => { launcherPath: "/missing/hopper", loaderArgsOverride: [], provider: HOPPER_PROVIDER_IDENTITY, + resourceScope: inspectionResources.owner, }, ); expect(resolved).toEqual({ @@ -99,6 +104,7 @@ describe("Hopper analysis profiles", () => { launcherPath, loaderArgsOverride: scenario.override, provider: HOPPER_PROVIDER_IDENTITY, + resourceScope: inspectionResources.owner, }, ); expect(resolved).toMatchObject({ @@ -128,6 +134,7 @@ describe("Hopper analysis profiles", () => { launcherPath: launcher, loaderArgsOverride: [], provider: HOPPER_PROVIDER_IDENTITY, + resourceScope: inspectionResources.owner, signal: controller.signal, }); diff --git a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts index 28d9791a9f..ecb76d4af1 100644 --- a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts +++ b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts @@ -27,6 +27,7 @@ import { linuxHopperReleaseCacheDirectory, retryLinuxHopperPublisherCleanup, } from "../../../src/application/LinuxHopperPublisher.js"; +import { ArtifactReaderFailure } from "../../../src/artifacts/ArtifactReader.js"; import { HOPPER_LINUX_PRIMARY_FINGERPRINT, type LinuxHopperRelease, @@ -108,6 +109,21 @@ const fixture = async (family: "deb" | "arch" = "deb") => { }; }; +const verifyFixtureArchive = async ( + f: Awaited>, + family: "deb" | "arch" = "deb", +) => { + const result = await f.publisher.verifyArchive( + family, + f.release, + f.archive, + undefined, + f.publicKey, + ); + if (!result.ok) throw result.error; + return result.value; +}; + const admitFixturePublisher = ( fingerprint: string, module: typeof signatures = signatures, @@ -173,16 +189,9 @@ describe("Linux signed package retention boundary", () => { }); vi.stubGlobal("fetch", network); try { - const verified = await f.publisher.verifyArchive( - family, - f.release, - f.archive, - undefined, - f.publicKey, - ); - if (!verified.ok) throw verified.error; + const verifiedArchive = await verifyFixtureArchive(f, family); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toEqual({ ok: true, value: undefined }); await f.publisher.close(); // A new instance alone still shares process-local cryptographic proofs. @@ -240,16 +249,9 @@ linux.each([ const f = await fixture(); admitFixturePublisher(f.fingerprint); try { - const verified = await f.publisher.verifyArchive( - "deb", - f.release, - f.archive, - undefined, - f.publicKey, - ); - if (!verified.ok) throw verified.error; + const verifiedArchive = await verifyFixtureArchive(f, "deb"); const retained = await f.publisher.retainVerifiedArchive( - verified.value, + verifiedArchive, f.archive, ); if (!retained.ok) throw retained.error; @@ -300,7 +302,7 @@ linux.each([ // An explicitly approved fresh package can repair this entry while // preserving the original evidence and keeping ordinary reads strict. expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toEqual({ ok: true, value: undefined }); const backups = (await readdir(f.publisher.cacheRoot)).filter((name) => name.startsWith(`${f.release.main_exe_hash}.backup-`), @@ -321,7 +323,7 @@ linux.each([ ok: true, }); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toEqual({ ok: true, value: undefined }); expect( (await readdir(f.publisher.cacheRoot)).filter((name) => @@ -345,16 +347,9 @@ linux.each([false, true])( | ReturnType | undefined; try { - const verified = await f.publisher.verifyArchive( - "deb", - f.release, - f.archive, - undefined, - f.publicKey, - ); - if (!verified.ok) throw verified.error; + const verifiedArchive = await verifyFixtureArchive(f, "deb"); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toEqual({ ok: true, value: undefined }); const staged = join(f.publisher.cacheRoot, "authenticated-fixture-stage"); await cp(f.cache, staged, { recursive: true }); @@ -399,7 +394,7 @@ linux.each([false, true])( }, ); retention = f.publisher.retainVerifiedArchive( - verified.value, + verifiedArchive, f.archive, controller.signal, ); @@ -450,16 +445,9 @@ linux("restores old evidence when replacement publication fails", async () => { const f = await fixture(); admitFixturePublisher(f.fingerprint); try { - const verified = await f.publisher.verifyArchive( - "deb", - f.release, - f.archive, - undefined, - f.publicKey, - ); - if (!verified.ok) throw verified.error; + const verifiedArchive = await verifyFixtureArchive(f, "deb"); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toEqual({ ok: true, value: undefined }); const broken = "{interrupted cache metadata"; await writeFile(join(f.cache, "release.json"), broken); @@ -479,7 +467,7 @@ linux("restores old evidence when replacement publication fails", async () => { syncBuiltinESMExports(); try { const result = await f.publisher.retainVerifiedArchive( - verified.value, + verifiedArchive, f.archive, ); expect(result).toMatchObject({ @@ -506,7 +494,7 @@ linux("restores old evidence when replacement publication fails", async () => { syncBuiltinESMExports(); } expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toEqual({ ok: true, value: undefined }); expect(await f.publisher.inspectLauncher(f.launcher)).toMatchObject({ ok: true, @@ -522,16 +510,9 @@ linux( const f = await fixture(); admitFixturePublisher(f.fingerprint); try { - const verified = await f.publisher.verifyArchive( - "deb", - f.release, - f.archive, - undefined, - f.publicKey, - ); - if (!verified.ok) throw verified.error; + const verifiedArchive = await verifyFixtureArchive(f, "deb"); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toEqual({ ok: true, value: undefined }); await writeFile(join(f.cache, "public-key.asc"), f.publicKey + "\n"); const metadata = await readFile(join(f.cache, "release.json")); @@ -539,7 +520,7 @@ linux( .spyOn(nativeCommands, "execFileOutput") .mockRejectedValue(new Error("Fixture native inspection unavailable")); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toMatchObject({ ok: false, error: { reason: "verification_unavailable" }, @@ -621,16 +602,9 @@ linux( }, ); try { - const verified = await f.publisher.verifyArchive( - "deb", - f.release, - f.archive, - undefined, - f.publicKey, - ); - if (!verified.ok) throw verified.error; + const verifiedArchive = await verifyFixtureArchive(f, "deb"); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toMatchObject({ ok: false, error: { reason: "cleanup_failed" } }); const input = { cacheDirectory: f.publisher.cacheRoot, @@ -889,16 +863,9 @@ portableTar( async () => { const f = await fixture("arch"); admitFixturePublisher(f.fingerprint); - const verified = await f.publisher.verifyArchive( - "arch", - f.release, - f.archive, - undefined, - f.publicKey, - ); - if (!verified.ok) throw verified.error; + const verifiedArchive = await verifyFixtureArchive(f, "arch"); expect( - await f.publisher.retainVerifiedArchive(verified.value, f.archive), + await f.publisher.retainVerifiedArchive(verifiedArchive, f.archive), ).toMatchObject({ ok: true }); let reachedMetadataClose!: () => void; @@ -964,6 +931,7 @@ linux( controller.abort(); throw new Error("Fixture cleanup unavailable after cancellation"); }); + let retryFailure: unknown; try { expect( await f.publisher.verifyArchive( @@ -977,10 +945,22 @@ linux( ok: false, error: { reason: "cleanup_failed" }, }); - await expect(retryLinuxHopperPublisherCleanup()).rejects.toMatchObject({ + try { + await retryLinuxHopperPublisherCleanup(); + } catch (cause: unknown) { + retryFailure = cause; + } + expect(retryFailure).toBeInstanceOf(ArtifactReaderFailure); + expect(retryFailure).toMatchObject({ cleanup: { resources: [expect.stringContaining("rea-hopper-publisher-")], }, + cleanupLeases: [expect.objectContaining({ released: false })], + }); + cleanup.mockRestore(); + await retryLinuxHopperPublisherCleanup(); + expect(retryFailure).toMatchObject({ + cleanupLeases: [expect.objectContaining({ released: true })], }); } finally { cleanup.mockRestore(); From 4d7de00d4cc9a95ec2b7a3123b04e01db2daafa0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:12:21 +0800 Subject: [PATCH 22/35] fix(native): coalesce Objective-C class imports by binding identity Join provider class observations only when selected-image identity and exact import bindings establish equivalence. Preserve distinct observations and unsupported coverage. Refs #5607 --- docs/native-investigation.md | 15 + .../EnhancedTools.objcClasses.test.ts | 446 ++++++++++++++++++ src/application/EnhancedTools.ts | 154 +++++- .../NativeDispatchMetadataInspection.ts | 40 +- src/contracts/enhancedInputs.ts | 21 +- src/contracts/enhancedToolContracts.ts | 4 +- src/contracts/toolOutputSchemaGroups.ts | 3 +- src/contracts/toolOutputSchemaPrimitives.ts | 27 ++ src/domain/native/objcSwiftMetadata.ts | 39 +- src/domain/symbolAnalysis.objcClasses.test.ts | 182 ++++++- src/domain/symbolAnalysis.ts | 196 +++++++- src/native/AppleDispatchMetadata.test.ts | 18 +- src/native/AppleDispatchMetadata.ts | 50 +- src/native/AppleMachoFixups.test.ts | 203 ++++++-- src/native/AppleMachoFixups.ts | 96 +++- src/native/AppleMachoSelection.test.ts | 59 ++- src/native/AppleMachoSelection.ts | 79 +++- .../AppleObjcClassImportBindings.test.ts | 168 +++++++ src/native/AppleObjcClassImportBindings.ts | 119 +++++ src/native/NativeMacOSProvider.ts | 2 + .../acceptance/nativeDispatchSnapshot.test.ts | 78 +++ ...4-uses-complete-inventories-inline.test.ts | 140 +++++- .../native/objcClassMetadataFallback.test.ts | 179 +++++++ tests/fixtures/nativeObjcClassImportMachO.ts | 69 +++ 24 files changed, 2287 insertions(+), 100 deletions(-) create mode 100644 src/application/EnhancedTools.objcClasses.test.ts create mode 100644 src/native/AppleObjcClassImportBindings.test.ts create mode 100644 src/native/AppleObjcClassImportBindings.ts create mode 100644 tests/boundary/providers/native/objcClassMetadataFallback.test.ts create mode 100644 tests/fixtures/nativeObjcClassImportMachO.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index b07fb53e1d..64417b7ed8 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -466,6 +466,21 @@ complete observation. that operation is unavailable; runtime failures remain failures rather than being replaced by symbol-only results. +`get_objc_classes` keeps names from the provider inventory. It coalesces rows +only when their exact selected-image pointer slots decode to the same class +import, including its binding stream, raw library ordinal, resolved library, weak status, and addend. A literal filter is +applied to the original names before that join; if the filter selects only an +alias, the output keeps that exact row. `identity_coverage` distinguishes a +complete selected-inventory scan from partial, unsupported, and unknown cases. +Recognized Mach-O formats outside the native metadata reader's support retain +the provider's class labels with unknown import identity; malformed headers and +read failures remain errors. +The join also requires matching whole-artifact digests and matching provider- +owned selected-image facts: container, byte range, full raw CPU type/subtype, +and FAT table digest when present. A missing or overridden loader selection +remains unknown; the result describes agreement on the selected artifact image, +not an atomic observation of a running process or whole-program class coverage. + ## Native desktop observation `observe_native_ui` captures one explicitly selected existing PID/window ID. diff --git a/src/application/EnhancedTools.objcClasses.test.ts b/src/application/EnhancedTools.objcClasses.test.ts new file mode 100644 index 0000000000..7552f4c44f --- /dev/null +++ b/src/application/EnhancedTools.objcClasses.test.ts @@ -0,0 +1,446 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + createAnalysisExecution, + type AnalysisOperation, + type AnalysisOperationPort, +} from "./AnalysisProvider.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { EnhancedTools } from "./EnhancedTools.js"; +import type { BinaryArchitecture } from "../domain/binaryTargetTypes.js"; +import { objcSwiftMetadataSchema } from "../domain/native/objcSwiftMetadata.js"; +import { err, ok } from "../domain/result.js"; +import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; +import { + AnalysisOutputError, + AnalysisCancelledError, +} from "../domain/analysisErrorCore.js"; +import { createAnalysisProfile } from "../domain/analysisProfile.js"; +import type { MachoTargetSelection } from "../domain/apple/machoTargetSelection.js"; + +const sha256 = "a".repeat(64); +const provider = { id: "native", name: "Native", version: "1" }; +const namesProvider = { id: "hopper", name: "Hopper", version: "2" }; +const selection = { + kind: "macho-selected-image-v1" as const, + container: "thin" as const, + offset: 0, + byte_length: 4096, + cpu_type: 0x0100000c, + cpu_subtype: 0x80000002, + container_table_sha256: null, +}; +const profile = createAnalysisProfile(namesProvider, { + target_selection: selection, +}); +const subject = { + sha256, + path: "/fixture", + format: "mach-o" as const, + architecture: "arm64" as const, +}; + +const createPort = ( + nativeOverrides: Partial<{ + sha256: string; + architecture: BinaryArchitecture; + names: Array<{ address: string; value: string }>; + nameLimitations: string[]; + metadataFailure: AnalysisError; + bindings: Array<{ + requested_address: string; + symbol: string; + binding_source: "chained" | "bind" | "weak" | "lazy"; + library_ordinal: number; + library: string | null; + weak: boolean; + addend: string; + }>; + targetSelection: MachoTargetSelection | null; + nameProfile: boolean; + }> = {}, +): AnalysisOperationPort => ({ + execute: async (operation: AnalysisOperation, parameters) => { + if (operation === "list_names") + return ok( + createAnalysisExecution( + nativeOverrides.names ?? [ + { address: "0x1000", value: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x1000", value: "OBJC_CLASS_$_Document" }, + ], + namesProvider, + { + subject, + limitations: nativeOverrides.nameLimitations ?? [], + ...(nativeOverrides.nameProfile === false + ? {} + : { analysisProfile: profile }), + }, + ), + ); + if (operation !== "inspect_native_dispatch_metadata") + return err( + new AnalysisCapabilityUnavailableError( + provider.id, + operation, + "Unavailable in fixture", + ), + ); + + expect(parameters).toMatchObject({ + max_records: 20_000, + objc_class_import_addresses: [ + ...new Set( + ( + nativeOverrides.names ?? [ + { address: "0x1000", value: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x1000", value: "OBJC_CLASS_$_Document" }, + ] + ).map(({ address }) => address), + ), + ], + objc_class_import_addresses_complete: true, + }); + if (nativeOverrides.metadataFailure !== undefined) + return err(nativeOverrides.metadataFailure); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + if (nativeOverrides.targetSelection !== null) + result.target_selection = nativeOverrides.targetSelection ?? selection; + result.objc_class_import_bindings = nativeOverrides.bindings ?? [ + { + requested_address: "0x1000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: false, + addend: "0", + }, + ]; + result.objc_class_import_binding_coverage = { + facet: "objc_class_import_bindings", + status: "complete", + reason: null, + examined: 1, + decoded: result.objc_class_import_bindings.length, + }; + return ok( + createAnalysisExecution( + { + target_sha256: nativeOverrides.sha256 ?? sha256, + target_architecture: nativeOverrides.architecture ?? "arm64", + provider, + analysis_profile_digest: null, + result, + }, + provider, + { + subject: { + ...subject, + ...(nativeOverrides.sha256 === undefined + ? {} + : { sha256: nativeOverrides.sha256 }), + ...(nativeOverrides.architecture === undefined + ? {} + : { architecture: nativeOverrides.architecture }), + }, + }, + ), + ); + }, +}); + +it.each([ + ["list_names", "omitted"], + ["list_names", "unbound"], + ["inspect_native_dispatch_metadata", "omitted"], + ["inspect_native_dispatch_metadata", "unbound"], +] as const)( + "retains class candidates when %s subject is %s", + async (operation, state) => { + const port = createPort(); + const tools = new EnhancedTools({ + execute: async (...args) => { + const result = await port.execute(...args); + if (!result.ok || args[0] !== operation) return result; + const value = { ...result.value }; + if (state === "omitted") delete value.subject; + else value.subject = null; + return ok(value); + }, + }); + await expect(tools.execute("get_objc_classes", {})).resolves.toMatchObject({ + ok: true, + value: { + count: 2, + classes: [ + { address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x1000", name: "OBJC_CLASS_$_Document" }, + ], + identity_coverage: { status: "unknown" }, + }, + }); + }, +); + +describe("Objective-C class import identity composition", () => { + it("joins only exact same-target bind slots and preserves literal filter selection", async () => { + const tools = new EnhancedTools(createPort()); + await expect(tools.execute("get_objc_classes", {})).resolves.toMatchObject({ + ok: true, + value: { + count: 1, + classes: [ + { + address: "0x1000", + name: "OBJC_CLASS_$_Document", + binding_identity: { + symbol: "_OBJC_CLASS_$_Document", + slot_addresses: ["0x1000"], + }, + }, + ], + identity_coverage: { status: "complete", reason: null }, + }, + }); + await expect( + tools.execute("get_objc_classes", { pattern: "$GOT" }), + ).resolves.toMatchObject({ + ok: true, + value: { + count: 1, + classes: [{ address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }], + }, + }); + }); + + it("keeps rows unjoined when the native observation has a different subject", async () => { + const tools = new EnhancedTools(createPort({ sha256: "b".repeat(64) })); + await expect(tools.execute("get_objc_classes", {})).resolves.toMatchObject({ + ok: true, + value: { + count: 2, + classes: [ + { address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x1000", name: "OBJC_CLASS_$_Document" }, + ], + identity_coverage: { status: "unknown" }, + }, + }); + }); + + it("keeps identity unknown when exact selected-image evidence is missing or mismatched", async () => { + const cases = [ + { nameProfile: false }, + { targetSelection: null }, + { targetSelection: { ...selection, cpu_type: 0x01000007 } }, + { targetSelection: { ...selection, cpu_subtype: 2 } }, + { + targetSelection: { + ...selection, + container: "fat" as const, + offset: 4096, + container_table_sha256: "b".repeat(64), + }, + }, + ]; + for (const overrides of cases) { + const tools = new EnhancedTools(createPort(overrides)); + await expect( + tools.execute("get_objc_classes", {}), + ).resolves.toMatchObject({ + ok: true, + value: { count: 2, identity_coverage: { status: "unknown" } }, + }); + } + }); + + it("marks identity coverage partial when the selected names inventory is limited", async () => { + const tools = new EnhancedTools( + createPort({ nameLimitations: ["provider capped the name inventory"] }), + ); + await expect(tools.execute("get_objc_classes", {})).resolves.toMatchObject({ + ok: true, + value: { + count: 1, + identity_coverage: { + status: "partial", + reason: "list_names: provider capped the name inventory", + }, + }, + }); + }); + + it("keeps names with unknown identity when binding inspection is unavailable", async () => { + const tools = new EnhancedTools( + createPort({ + metadataFailure: new AnalysisCapabilityUnavailableError( + provider.id, + "inspect_native_dispatch_metadata", + "No static Mach-O reader", + ), + }), + ); + await expect(tools.execute("get_objc_classes", {})).resolves.toMatchObject({ + ok: true, + value: { + count: 2, + identity_coverage: { status: "unknown" }, + }, + }); + }); + + it("preserves non-capability binding failures", async () => { + const failure = new AnalysisOutputError( + "inspect_native_dispatch_metadata", + "fixture failed", + ); + const tools = new EnhancedTools(createPort({ metadataFailure: failure })); + await expect(tools.execute("get_objc_classes", {})).resolves.toEqual({ + ok: false, + error: failure, + }); + }); +}); + +it("preserves same-named rows whose exact import slots have different identities", async () => { + const addresses = ["0x1000", "0x2000", "0x3000", "0x4000"]; + const name = "OBJC_CLASS_$_Document"; + const tools = new EnhancedTools( + createPort({ + names: [ + ...addresses.map((address) => ({ address, value: name })), + { address: "0x1000", value: name }, + ], + bindings: [ + { + requested_address: "0x1000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: false, + addend: "0", + }, + { + requested_address: "0x2000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/Frameworks/UIKit.framework/UIKit", + weak: false, + addend: "0", + }, + { + requested_address: "0x3000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: true, + addend: "0", + }, + { + requested_address: "0x4000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: false, + addend: "1", + }, + ], + }), + ); + + await expect(tools.execute("get_objc_classes", {})).resolves.toMatchObject({ + ok: true, + value: { + count: 4, + classes: addresses.map((address, index) => ({ + address, + name, + binding_identity: { + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: + index === 1 + ? "/System/Library/Frameworks/UIKit.framework/UIKit" + : "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: index === 2, + addend: index === 3 ? "1" : "0", + slot_addresses: [address], + }, + })), + identity_coverage: { status: "complete", reason: null }, + }, + }); +}); + +describe("EnhancedTools cancellation ownership", () => { + it("preserves the caller reason at admission", async () => { + const reason = { request: "cancel-before-observation" }; + const controller = new AbortController(); + controller.abort(reason); + const execute = vi.fn(createPort().execute); + const tools = new EnhancedTools({ execute }); + + const result = await tools.execute( + "get_objc_classes", + {}, + controller.signal, + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Pre-cancelled observation was accepted"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(execute).not.toHaveBeenCalled(); + }); + + it("preserves a post-success caller reason without replacing provider failures", async () => { + const reason = { request: "cancel-after-provider-success" }; + const controller = new AbortController(); + const successfulPort = createPort(); + const successTools = new EnhancedTools({ + execute: async (...args) => { + const result = await successfulPort.execute(...args); + if (args[0] === "list_names" && result.ok) controller.abort(reason); + return result; + }, + }); + + const cancelled = await successTools.execute( + "get_objc_classes", + {}, + controller.signal, + ); + expect(cancelled.ok).toBe(false); + if (cancelled.ok) + throw new Error("Post-cancelled observation was accepted"); + expect(cancelled.error).toBeInstanceOf(AnalysisCancelledError); + expect(cancelled.error.cause).toBe(reason); + + const physicalFailure = new AnalysisOutputError( + "list_names", + "fixture read failed", + ); + const failingController = new AbortController(); + const failureTools = new EnhancedTools({ + execute: async (...args) => { + if (args[0] === "list_names") { + failingController.abort(reason); + return err(physicalFailure); + } + return successfulPort.execute(...args); + }, + }); + const failed = await failureTools.execute( + "get_objc_classes", + {}, + failingController.signal, + ); + expect(failed).toEqual({ ok: false, error: physicalFailure }); + }); +}); diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index dffef765f3..8f0208e2b6 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -9,6 +9,7 @@ import type { } from "./AnalysisProvider.js"; import { enhancedInputSchemas, + MAX_ENHANCED_RECORDS, type CallRelationshipDirection, type EnhancedToolName, } from "../contracts/enhancedInputs.js"; @@ -30,15 +31,24 @@ import { import { err, ok, type Result } from "../domain/result.js"; import { categorizeSwiftTypes, - discoverObjcClasses, discoverObjcProtocols, + projectObjcClassCandidates, + selectObjcClassCandidates, } from "../domain/symbolAnalysis.js"; import { inspectNativeDispatch } from "./native/NativeDispatchMetadataInspection.js"; import { traceNativeValues } from "./native/NativeValueTrace.js"; import { traceNativeUiAction } from "./native/NativeUiActionTrace.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { addressedValueSchema } from "../domain/native/nativeInventory.js"; +import { nativeDispatchMetadataResultSchema } from "../domain/native/objcSwiftMetadata.js"; +import { isMachOFormat } from "../domain/binaryTargetTypes.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { + machoTargetSelectionMatchesArchitecture, + machoTargetSelectionMatchesFormat, + profileMachoTargetSelection, + sameMachoTargetSelection, +} from "../domain/apple/machoTargetSelection.js"; import { invalidEnhancedInput, @@ -187,6 +197,8 @@ export class EnhancedTools { return this.#inspectNativeDispatchMetadata( call.input.max_records, signal, + call.input.objc_class_import_addresses, + call.input.objc_class_import_addresses_complete, ); case "get_objc_classes": return this.#objcClasses(call.input.pattern, signal); @@ -286,15 +298,137 @@ export class EnhancedTools { } async #objcClasses(pattern: string, signal?: AbortSignal): EnhancedResult { - const names = await this.#allAddressed("list_names", signal); - return names.ok ? ok(discoverObjcClasses(names.value, pattern)) : names; + const namesExecution = await this.#observe("list_names", {}, signal); + if (!namesExecution.ok) return namesExecution; + const names = this.#parseAddressed( + "list_names", + namesExecution.value.result, + ); + if (!names.ok) return names; + const candidates = selectObjcClassCandidates(names.value, pattern); + if (candidates.length === 0) + return ok( + projectObjcClassCandidates(candidates, { + bindings: [], + coverage: { status: "not_applicable", reason: null }, + }), + ); + + const addresses = [...new Set(candidates.map(({ address }) => address))]; + const scannedAddresses = addresses.slice(0, MAX_ENHANCED_RECORDS); + const bindingsExecution = await this.#observe( + "inspect_native_dispatch_metadata", + { + max_records: MAX_ENHANCED_RECORDS, + objc_class_import_addresses: scannedAddresses, + objc_class_import_addresses_complete: + addresses.length <= MAX_ENHANCED_RECORDS, + }, + signal, + ); + if (!bindingsExecution.ok) { + if (bindingsExecution.error._tag === "AnalysisCancelledError") + return bindingsExecution; + if (bindingsExecution.error._tag !== "AnalysisCapabilityUnavailableError") + return bindingsExecution; + return ok( + projectObjcClassCandidates(candidates, { + bindings: [], + coverage: { + status: "unknown", + reason: `Native class-import binding evidence is unavailable (${bindingsExecution.error._tag}).`, + }, + }), + ); + } + + const native = nativeDispatchMetadataResultSchema.safeParse( + bindingsExecution.value.result, + ); + const nameSubject = namesExecution.value.subject; + const bindingSubject = bindingsExecution.value.subject; + const namesSelection = profileMachoTargetSelection( + namesExecution.value.analysisProfile, + ); + const nativeSelection = native.success + ? native.data.result.target_selection + : undefined; + if ( + !native.success || + nameSubject === null || + nameSubject === undefined || + bindingSubject === null || + bindingSubject === undefined || + nameSubject.sha256 !== bindingSubject.sha256 || + nameSubject.architecture === undefined || + bindingSubject.architecture === undefined || + nameSubject.architecture !== bindingSubject.architecture || + !isMachOFormat(nameSubject.format) || + nameSubject.format !== bindingSubject.format || + native.data.target_sha256 !== nameSubject.sha256 || + native.data.target_architecture !== nameSubject.architecture || + native.data.provider.id !== bindingsExecution.value.provider.id || + namesExecution.value.analysisProfile?.provider.id !== + namesExecution.value.provider.id || + nativeSelection === undefined || + !machoTargetSelectionMatchesArchitecture( + nativeSelection, + nameSubject.architecture, + ) || + !machoTargetSelectionMatchesFormat(nativeSelection, nameSubject.format) || + !sameMachoTargetSelection(namesSelection, nativeSelection) || + native.data.result.objc_class_import_bindings === undefined || + native.data.result.objc_class_import_binding_coverage === undefined + ) + return ok( + projectObjcClassCandidates(candidates, { + bindings: [], + coverage: { + status: "unknown", + reason: + "Native class-import facts did not prove matching artifact, architecture, and complete producer identity.", + }, + }), + ); + + const coverage = native.data.result.objc_class_import_binding_coverage; + const inventoryLimitations = namesExecution.value.limitations; + const inventoryIncomplete = inventoryLimitations.length > 0; + const coverageReason = [ + coverage.reason, + ...inventoryLimitations.map((limitation) => `list_names: ${limitation}`), + ] + .filter((reason): reason is string => reason !== null) + .join("; "); + return ok( + projectObjcClassCandidates(candidates, { + bindings: native.data.result.objc_class_import_bindings, + coverage: { + status: + coverage.status === "not_requested" + ? "unknown" + : coverage.status === "complete" && inventoryIncomplete + ? "partial" + : coverage.status, + reason: coverageReason.length === 0 ? null : coverageReason, + }, + }), + ); } #inspectNativeDispatchMetadata( maxRecords: number, signal?: AbortSignal, + objcClassImportAddresses?: readonly string[], + objcClassImportAddressesComplete?: boolean, ): EnhancedResult { - return inspectNativeDispatch(this.analysis, maxRecords, signal); + return inspectNativeDispatch( + this.analysis, + maxRecords, + signal, + objcClassImportAddresses, + objcClassImportAddressesComplete, + ); } async #objcProtocols(signal?: AbortSignal): EnhancedResult { @@ -677,7 +811,11 @@ export class EnhancedTools { ) { const result = await this.#call(tool, {}, signal); if (!result.ok) return result; - const parsed = z.array(addressedValueSchema).safeParse(result.value); + return this.#parseAddressed(tool, result.value); + } + + #parseAddressed(tool: "list_names" | "list_procedures", value: JsonValue) { + const parsed = z.array(addressedValueSchema).safeParse(value); return parsed.success ? ok( parsed.data.map(({ address, value: name, objc_class_label }) => ({ @@ -715,13 +853,15 @@ export class EnhancedTools { signal?: AbortSignal, selectorOrigin?: ExecutionOptions["selectorOrigin"], ): Promise> { - if (isAborted(signal)) return err(new AnalysisCancelledError(name)); + if (isAborted(signal)) + return err(new AnalysisCancelledError(name, { cause: signal?.reason })); const execution = await this.analysis.execute(name, arguments_, { ...(signal === undefined ? {} : { signal }), ...(selectorOrigin === undefined ? {} : { selectorOrigin }), }); if (!execution.ok) return execution; - if (isAborted(signal)) return err(new AnalysisCancelledError(name)); + if (isAborted(signal)) + return err(new AnalysisCancelledError(name, { cause: signal?.reason })); return execution; } } diff --git a/src/application/native/NativeDispatchMetadataInspection.ts b/src/application/native/NativeDispatchMetadataInspection.ts index c27898bc6c..f71cc39fdf 100644 --- a/src/application/native/NativeDispatchMetadataInspection.ts +++ b/src/application/native/NativeDispatchMetadataInspection.ts @@ -21,10 +21,22 @@ export const inspectNativeDispatch = async ( analysis: AnalysisOperationPort, maxRecords: number, signal?: AbortSignal, + objcClassImportAddresses?: readonly string[], + objcClassImportAddressesComplete?: boolean, ): EnhancedResult => { + const input = { + max_records: maxRecords, + ...(objcClassImportAddresses === undefined + ? {} + : { + objc_class_import_addresses: [...objcClassImportAddresses], + objc_class_import_addresses_complete: + objcClassImportAddressesComplete ?? true, + }), + }; const binaryMetadata = await analysis.execute( NATIVE_DISPATCH_METADATA_SOURCES.binary, - { max_records: maxRecords }, + input, signal === undefined ? {} : { signal }, ); if (binaryMetadata.ok) @@ -59,20 +71,32 @@ export const inspectNativeDispatch = async ( "provider returned an invalid inventory", ), ); + const result = inspectNativeDispatchMetadata( + names.data.map((entry) => ({ + address: entry.address, + name: "value" in entry ? entry.value : entry.name, + })), + maxRecords, + ); + if (objcClassImportAddresses !== undefined) { + result.objc_class_import_bindings = []; + result.objc_class_import_binding_coverage = { + facet: "objc_class_import_bindings", + status: "unsupported", + reason: "Native Mach-O pointer binding inspection is unavailable.", + examined: 0, + decoded: 0, + }; + } return ok( jsonValueSchema.parse( nativeDispatchMetadataResultSchema.parse({ target_sha256: execution.value.subject?.sha256 ?? null, + target_architecture: execution.value.subject?.architecture ?? null, provider: execution.value.provider, analysis_profile_digest: execution.value.analysisProfile?.digest ?? null, - result: inspectNativeDispatchMetadata( - names.data.map((entry) => ({ - address: entry.address, - name: "value" in entry ? entry.value : entry.name, - })), - maxRecords, - ), + result, }), ), ); diff --git a/src/contracts/enhancedInputs.ts b/src/contracts/enhancedInputs.ts index 63d5a1682f..029947485d 100644 --- a/src/contracts/enhancedInputs.ts +++ b/src/contracts/enhancedInputs.ts @@ -6,6 +6,7 @@ const traceLiteralInputSchema = z.strictObject({ }); export const DEFAULT_CALL_RELATIONSHIP_DIRECTION = "forward"; +export const MAX_ENHANCED_RECORDS = 20_000; const callRelationshipDirectionSchema = z .enum(["forward", "backward"]) .default(DEFAULT_CALL_RELATIONSHIP_DIRECTION); @@ -17,7 +18,25 @@ export type CallRelationshipDirection = z.output< export const enhancedInputSchemas = { trace_native_values: nativeValueTraceInputSchema, inspect_native_dispatch_metadata: z.strictObject({ - max_records: z.number().int().min(1).max(20_000).default(5_000), + max_records: z + .number() + .int() + .min(1) + .max(MAX_ENHANCED_RECORDS) + .default(5_000), + objc_class_import_addresses: z + .array(z.string()) + .max(MAX_ENHANCED_RECORDS) + .describe( + "Optional exact pointer-slot addresses from the selected name inventory; used to report decoded Objective-C class import bindings.", + ) + .optional(), + objc_class_import_addresses_complete: z + .boolean() + .describe( + "Whether the candidate addresses include every selected class-label row; false keeps binding coverage partial when the existing record budget truncates the scan.", + ) + .optional(), }), get_objc_classes: z.strictObject({ pattern: z.string().default("") }), get_objc_protocols: z.strictObject({}), diff --git a/src/contracts/enhancedToolContracts.ts b/src/contracts/enhancedToolContracts.ts index 9b124f8188..3ecc2b9a7c 100644 --- a/src/contracts/enhancedToolContracts.ts +++ b/src/contracts/enhancedToolContracts.ts @@ -27,12 +27,12 @@ const enhanced = ( export const ENHANCED_TOOL_CONTRACTS = [ enhanced( "inspect_native_dispatch_metadata", - "Inspect the bound provider's name inventory for Objective-C class/method symbols and Swift mangled symbols, returning at most max_records decoded names. Includes target SHA-256 when available, provider identity/version, analysis-profile digest, facet coverage, and per-record Evidence. This is symbol-level evidence, not runtime metadata: ivar layouts, protocol conformances, witness/class tables, and relative pointers are explicitly marked unsupported unless a provider supplies those decoders. The provider controls how its name inventory is acquired.", + "Inspect the bound provider's name inventory for Objective-C class/method symbols and Swift mangled symbols, returning at most max_records decoded names. Optional objc_class_import_addresses request exact pointer-slot binding facts for those addresses; they are decoded only from the selected Mach-O slice and retain partial or unsupported coverage when not established. Includes target SHA-256 when available, provider identity/version, analysis-profile digest, facet coverage, and per-record Evidence. This is static symbol and metadata evidence, not runtime metadata: ivar layouts, protocol conformances, witness/class tables, and relative pointers are explicitly marked unsupported unless a provider supplies those decoders. The provider controls how its name inventory is acquired.", enhancedInputSchemas.inspect_native_dispatch_metadata, ), enhanced( "get_objc_classes", - "Discover and deduplicate Objective-C class labels, optionally filtering by literal substring.", + "Discover Objective-C class labels, optionally filtering by literal substring. Rows are coalesced only when exact selected-image pointer-fixup evidence proves that their own addresses bind the same imported class symbol, binding stream, raw library ordinal and resolved library, weak status, and addend. Rows without that evidence remain separate, with identity coverage reported as unknown, partial, or unsupported.", enhancedInputSchemas.get_objc_classes, ), enhanced( diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index cd1864c99c..1579f2342f 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -88,6 +88,7 @@ import { lifecycleResultOf, nullableText, procedureInfoOutput, + objcClassDiscoveryOutput, evidenceResultOf, segmentOutput, sessionProvider, @@ -309,7 +310,7 @@ export const enhancedOutputSchemas: Readonly> = { ), trace_native_values: evidenceResultOf(nativeValueTraceSchema), trace_native_ui_action: evidenceResultOf(nativeInvestigationTraceSchema), - get_objc_classes: symbolDiscoveryOutput("classes"), + get_objc_classes: objcClassDiscoveryOutput, get_objc_protocols: symbolDiscoveryOutput("protocols"), batch_decompile: evidenceResultOf( z.object({ diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index 058388b201..1e796ecc0e 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -23,6 +23,7 @@ import { import { analysisErrorProjectionSchema } from "./errorSchemas.js"; import { relationshipCoverageSchema } from "../domain/relationshipCoverage.js"; import { ZIP_PACKAGE_FORMATS } from "../domain/zipPackageFormat.js"; +import { objcClassImportIdentitySchema } from "../domain/native/objcSwiftMetadata.js"; /** Specialize the single normalized result inside a complete Evidence record. */ export const evidenceResultOf = (schema: Schema) => @@ -606,6 +607,32 @@ export const symbolDiscoveryOutput = (property: "classes" | "protocols") => }), ); +export const objcClassDiscoveryOutput = evidenceResultOf( + z.object({ + count: z.number().int().min(0), + classes: z.array( + addressedEntry.extend({ + binding_identity: objcClassImportIdentitySchema + .extend({ + slot_addresses: z.array(z.string().min(1)).min(1), + }) + .optional(), + }), + ), + identity_coverage: z.strictObject({ + status: z.enum([ + "complete", + "partial", + "unsupported", + "not_requested", + "unknown", + "not_applicable", + ]), + reason: z.string().nullable(), + }), + }), +); + export const graphNode = z.discriminatedUnion("status", [ z.object({ address: z.string(), diff --git a/src/domain/native/objcSwiftMetadata.ts b/src/domain/native/objcSwiftMetadata.ts index 85649ece29..546a801aca 100644 --- a/src/domain/native/objcSwiftMetadata.ts +++ b/src/domain/native/objcSwiftMetadata.ts @@ -1,8 +1,13 @@ import { z } from "zod"; -import { isMachOFormat, type BinaryTarget } from "../binaryTargetTypes.js"; +import { + BINARY_ARCHITECTURES, + isMachOFormat, + type BinaryTarget, +} from "../binaryTargetTypes.js"; import { providerSchema } from "../evidence.js"; import { isSwiftMangledSymbolName } from "../swiftSymbolClassification.js"; import { nativeCallReferenceSchema } from "./nativeCallReference.js"; +import { machoTargetSelectionSchema } from "../apple/machoTargetSelection.js"; import { compareUnicodeCodePoints, @@ -248,6 +253,33 @@ export const nativeMetadataCoverageSchema = z.strictObject({ decoded: z.number().int().nonnegative(), }); +/** Exact Mach-O import binding observed at a caller-selected pointer slot. */ +export const objcClassImportBindingSchema = z.strictObject({ + requested_address: z.string().min(1), + symbol: z.string().min(1), + binding_source: z.enum(["chained", "bind", "weak", "lazy"]), + library_ordinal: z.number().int().min(-3).max(Number.MAX_SAFE_INTEGER), + library: z.string().nullable(), + weak: z.boolean(), + addend: z.string().regex(/^-?\d+$/u), +}); +export const objcClassImportIdentitySchema = objcClassImportBindingSchema.omit({ + requested_address: true, +}); +export const objcClassImportBindingCoverageSchema = z.strictObject({ + facet: z.literal("objc_class_import_bindings"), + status: z.enum(["complete", "partial", "unsupported", "not_requested"]), + reason: z.string().nullable(), + examined: z.number().int().nonnegative(), + decoded: z.number().int().nonnegative(), +}); +export type ObjcClassImportBinding = z.infer< + typeof objcClassImportBindingSchema +>; +export type ObjcClassImportBindingCoverage = z.infer< + typeof objcClassImportBindingCoverageSchema +>; + /** Swift declaration kind. */ export const swiftDeclKindSchema = z.enum([ "class", @@ -316,6 +348,10 @@ export const objcSwiftMetadataSchema = z.strictObject({ swift_conformances: z.array(swiftConformanceSchema).default([]), swift_dispatch_slots: z.array(swiftDispatchSlotSchema).default([]), swift_symbols: z.array(swiftSymbolSchema).default([]), + objc_class_import_bindings: z.array(objcClassImportBindingSchema).optional(), + objc_class_import_binding_coverage: + objcClassImportBindingCoverageSchema.optional(), + target_selection: machoTargetSelectionSchema.optional(), relative_pointers: z.array(relativePointerSchema).default([]), coverage: z.array(nativeMetadataCoverageSchema).default([]), db_save_result: dbSaveResultSchema.nullable(), @@ -328,6 +364,7 @@ export const nativeDispatchMetadataResultSchema = z.strictObject({ .string() .regex(/^[a-f0-9]{64}$/u) .nullable(), + target_architecture: z.enum(BINARY_ARCHITECTURES).nullable().optional(), provider: z.strictObject({ id: z.string().min(1), name: z.string().min(1), diff --git a/src/domain/symbolAnalysis.objcClasses.test.ts b/src/domain/symbolAnalysis.objcClasses.test.ts index d019ae4118..1b0aa13745 100644 --- a/src/domain/symbolAnalysis.objcClasses.test.ts +++ b/src/domain/symbolAnalysis.objcClasses.test.ts @@ -1,6 +1,20 @@ import { describe, expect, it } from "vitest"; -import { discoverObjcClasses } from "./symbolAnalysis.js"; +import { + discoverObjcClasses, + projectObjcClassCandidates, +} from "./symbolAnalysis.js"; + +const completeCoverage = { status: "complete" as const, reason: null }; +const classImport = { + requested_address: "0x1000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "bind" as const, + library_ordinal: 1, + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: false, + addend: "0", +}; describe("Objective-C class label discovery", () => { it("does not reinterpret a provider-specific label without an owner tag", () => { @@ -9,7 +23,11 @@ describe("Objective-C class label discovery", () => { [{ address: "0x1", name: "__objc_class_HuntRoot_class" }], "Hunt", ), - ).toEqual({ count: 0, classes: [] }); + ).toEqual({ + count: 0, + classes: [], + identity_coverage: { status: "not_applicable", reason: null }, + }); }); it("uses provider-tagged stripped class labels and filters their exact text", () => { @@ -39,10 +57,170 @@ describe("Objective-C class label discovery", () => { address, name, })), + identity_coverage: { + status: "unknown", + reason: "Exact class-import binding identity is unavailable.", + }, }); expect(discoverObjcClasses(names, "__objc_class_HuntRoot_class")).toEqual({ count: 1, classes: [{ address: "0x1", name: "__objc_class_HuntRoot_class" }], + identity_coverage: { + status: "unknown", + reason: "Exact class-import binding identity is unavailable.", + }, + }); + }); + + it("coalesces only selected rows whose own addresses prove the same import", () => { + const alias = { address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }; + const canonical = { address: "0x1000", name: "OBJC_CLASS_$_Document" }; + const unrelated = { address: "0x2000", name: "OBJC_CLASS_$_Document" }; + const scan = { + bindings: [classImport], + coverage: completeCoverage, + }; + + expect( + projectObjcClassCandidates([alias, canonical, unrelated], scan), + ).toEqual({ + count: 2, + classes: [ + { + ...canonical, + binding_identity: { + symbol: classImport.symbol, + binding_source: classImport.binding_source, + library_ordinal: classImport.library_ordinal, + library: classImport.library, + weak: false, + addend: "0", + slot_addresses: ["0x1000"], + }, + }, + unrelated, + ], + identity_coverage: completeCoverage, + }); + expect(discoverObjcClasses([alias, canonical], "$GOT", scan)).toEqual({ + count: 1, + classes: [ + { + ...alias, + binding_identity: { + symbol: classImport.symbol, + binding_source: classImport.binding_source, + library_ordinal: classImport.library_ordinal, + library: classImport.library, + weak: false, + addend: "0", + slot_addresses: ["0x1000"], + }, + }, + ], + identity_coverage: completeCoverage, + }); + }); + + it("does not join same-name rows without exact slot evidence", () => { + const alias = { address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }; + const canonical = { address: "0x2000", name: "OBJC_CLASS_$_Document" }; + expect( + projectObjcClassCandidates([alias, canonical], { + bindings: [classImport], + coverage: completeCoverage, + }), + ).toMatchObject({ count: 2, classes: [alias, canonical] }); + }); + + it("keeps rows separate when equal symbols have different bind identities", () => { + const rows = [ + { address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x2000", name: "OBJC_CLASS_$_Document$GOT" }, + ]; + expect( + projectObjcClassCandidates(rows, { + bindings: [ + classImport, + { + ...classImport, + requested_address: "0x2000", + library: "/other/framework.dylib", + }, + ], + coverage: completeCoverage, + }), + ).toMatchObject({ count: 2, classes: rows }); + }); + + it("keeps same-named candidates at distinct addresses until bind identity is known", () => { + const rows = [ + { address: "0x1000", name: "OBJC_CLASS_$_Document" }, + { address: "0x1000", name: "OBJC_CLASS_$_Document" }, + { address: "0x2000", name: "OBJC_CLASS_$_Document" }, + ]; + expect(discoverObjcClasses(rows, "")).toEqual({ + count: 2, + classes: [rows[0], rows[2]], + identity_coverage: { + status: "unknown", + reason: "Exact class-import binding identity is unavailable.", + }, }); }); }); + +it("keeps equal symbols separate when only their lookup ordinals differ", () => { + const rows = [ + { address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x2000", name: "OBJC_CLASS_$_Document$GOT" }, + ]; + expect( + discoverObjcClasses(rows, "", { + bindings: [ + { ...classImport, library_ordinal: -1, library: null }, + { + ...classImport, + requested_address: "0x2000", + library_ordinal: -2, + library: null, + }, + ], + coverage: completeCoverage, + }), + ).toMatchObject({ + count: 2, + classes: [ + { ...rows[0], binding_identity: { library_ordinal: -1 } }, + { ...rows[1], binding_identity: { library_ordinal: -2 } }, + ], + }); +}); + +it("keeps regular self-scope and weak rebinding records separate", () => { + const rows = [ + { address: "0x1000", name: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x2000", name: "OBJC_CLASS_$_Document$GOT" }, + ]; + expect( + discoverObjcClasses(rows, "", { + bindings: [ + { ...classImport, library_ordinal: 0, library: null }, + { + ...classImport, + requested_address: "0x2000", + binding_source: "weak", + library_ordinal: 0, + library: null, + }, + ], + coverage: completeCoverage, + }), + ).toMatchObject({ + count: 2, + classes: [ + { ...rows[0], binding_identity: { binding_source: "bind" } }, + { ...rows[1], binding_identity: { binding_source: "weak" } }, + ], + }); +}); diff --git a/src/domain/symbolAnalysis.ts b/src/domain/symbolAnalysis.ts index 792c09771e..f23e1ab8e3 100644 --- a/src/domain/symbolAnalysis.ts +++ b/src/domain/symbolAnalysis.ts @@ -1,5 +1,9 @@ import type { JsonValue } from "./jsonValue.js"; import type { AddressedName } from "./hopperValues.js"; +import type { + ObjcClassImportBinding, + ObjcClassImportBindingCoverage, +} from "./native/objcSwiftMetadata.js"; import { classifySwiftSymbol, isSwiftMangledSymbolName, @@ -18,12 +22,25 @@ const LEGACY_OBJC_CLASS_PREFIX = "l_OBJC_CLASS_" as const; */ const OBJC_PROTOCOL_MARKER = "OBJC_PROTOCOL_$_" as const; -/** Select and deduplicate Objective-C class labels. */ -export const discoverObjcClasses = ( +export interface ObjcClassDiscoveryCoverage { + readonly status: + | ObjcClassImportBindingCoverage["status"] + | "unknown" + | "not_applicable"; + readonly reason: string | null; +} + +export interface ObjcClassImportBindingScan { + readonly bindings: readonly ObjcClassImportBinding[]; + readonly coverage: ObjcClassDiscoveryCoverage; +} + +/** Select class labels and apply the caller's literal filter before identity joins. */ +export const selectObjcClassCandidates = ( names: readonly AddressedName[], pattern: string, -): JsonValue => { - const classes = uniqueByName( +): AddressedName[] => + uniqueByAddressAndName( names .filter( ({ name, objc_class_label }) => @@ -35,12 +52,43 @@ export const discoverObjcClasses = ( ) .filter(({ name }) => pattern.length === 0 || name.includes(pattern)), ); + +/** Project selected class labels, coalescing only exact proven import identities. */ +export const projectObjcClassCandidates = ( + candidates: readonly AddressedName[], + bindingScan: ObjcClassImportBindingScan = { + bindings: [], + coverage: { + status: candidates.length === 0 ? "not_applicable" : "unknown", + reason: + candidates.length === 0 + ? null + : "Exact class-import binding identity is unavailable.", + }, + }, +): JsonValue => { + const classes = coalesceBoundClassAliases(candidates, bindingScan.bindings); return { count: classes.length, - classes: classes.map(toJsonEntry), + classes: classes.map(toObjcClassEntry), + identity_coverage: { + status: bindingScan.coverage.status, + reason: bindingScan.coverage.reason, + }, }; }; +/** Select, then project Objective-C class labels for provider-neutral callers. */ +export const discoverObjcClasses = ( + names: readonly AddressedName[], + pattern: string, + bindingScan?: ObjcClassImportBindingScan, +): JsonValue => + projectObjcClassCandidates( + selectObjcClassCandidates(names, pattern), + bindingScan, + ); + /** Select and deduplicate Objective-C and Swift protocol labels. */ export const discoverObjcProtocols = ( names: readonly AddressedName[], @@ -185,6 +233,144 @@ const uniqueByName = (entries: readonly AddressedName[]): AddressedName[] => { }); }; +/** Keep same-named rows at distinct locations until slot identity is examined. */ +const uniqueByAddressAndName = ( + entries: readonly AddressedName[], +): AddressedName[] => { + const seen = new Set(); + return entries.filter(({ address, name }) => { + const key = JSON.stringify([address, name]); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +}; + +interface ObjcClassOutputEntry extends AddressedName { + readonly binding_identity?: Omit< + ObjcClassImportBinding, + "requested_address" + > & { + readonly slot_addresses: readonly string[]; + }; +} + +const coalesceBoundClassAliases = ( + candidates: readonly AddressedName[], + bindings: readonly ObjcClassImportBinding[], +): ObjcClassOutputEntry[] => { + const bindingsByAddress = new Map( + bindings.map((binding) => [binding.requested_address, binding]), + ); + const bindingKey = (binding: ObjcClassImportBinding) => + JSON.stringify([ + binding.symbol, + binding.binding_source, + binding.library_ordinal, + binding.library, + binding.weak, + binding.addend, + ]); + const groupedBindings = new Map(); + for (const binding of bindings) { + const key = bindingKey(binding); + const existing = groupedBindings.get(key); + if (existing === undefined) groupedBindings.set(key, [binding]); + else existing.push(binding); + } + + const groupedCandidates = new Map(); + const ordered: Array = []; + for (const candidate of candidates) { + const binding = bindingsByAddress.get(candidate.address); + if (binding === undefined) { + ordered.push(candidate); + continue; + } + const key = bindingKey(binding); + let group = groupedCandidates.get(key); + if (group === undefined) { + group = []; + groupedCandidates.set(key, group); + ordered.push({ group: key }); + } + group.push(candidate); + } + + const output: ObjcClassOutputEntry[] = []; + for (const entry of ordered) { + if ("group" in entry) { + const key = entry.group; + const facts = groupedBindings.get(key) ?? []; + const group = groupedCandidates.get(key) ?? []; + const identity = facts[0]; + const canonicalName = + identity === undefined + ? undefined + : classNameFromImport(identity.symbol); + const representative = + (canonicalName === undefined + ? undefined + : group.find( + ({ name }) => classNameFromDisplayName(name) === canonicalName, + )) ?? group[0]; + if (representative === undefined || identity === undefined) continue; + output.push({ + ...representative, + binding_identity: { + symbol: identity.symbol, + binding_source: identity.binding_source, + library_ordinal: identity.library_ordinal, + library: identity.library, + weak: identity.weak, + addend: identity.addend, + slot_addresses: [ + ...new Set(facts.map(({ requested_address }) => requested_address)), + ], + }, + }); + } else { + const candidate = entry; + output.push(candidate); + } + } + return output; +}; + +const classNameFromImport = (symbol: string): string | undefined => { + const canonical = symbol.startsWith("_") ? symbol.slice(1) : symbol; + return canonical.startsWith(OBJC_CLASS_MARKER) && + canonical.length > OBJC_CLASS_MARKER.length + ? canonical + : undefined; +}; + +const classNameFromDisplayName = (name: string): string | undefined => + name.startsWith(OBJC_CLASS_MARKER) && name.length > OBJC_CLASS_MARKER.length + ? name + : name.startsWith(`_${OBJC_CLASS_MARKER}`) && + name.length > OBJC_CLASS_MARKER.length + 1 + ? name.slice(1) + : undefined; + +const toObjcClassEntry = (entry: ObjcClassOutputEntry): JsonValue => ({ + address: entry.address, + name: entry.name, + ...(entry.binding_identity === undefined + ? {} + : { + binding_identity: { + symbol: entry.binding_identity.symbol, + binding_source: entry.binding_identity.binding_source, + library_ordinal: entry.binding_identity.library_ordinal, + library: entry.binding_identity.library, + weak: entry.binding_identity.weak, + addend: entry.binding_identity.addend, + slot_addresses: [...entry.binding_identity.slot_addresses], + }, + }), +}); + const toJsonEntry = ({ address, name }: AddressedName): JsonValue => ({ address, name, diff --git a/src/native/AppleDispatchMetadata.test.ts b/src/native/AppleDispatchMetadata.test.ts index 3a51c8a1f2..5bff9c1388 100644 --- a/src/native/AppleDispatchMetadata.test.ts +++ b/src/native/AppleDispatchMetadata.test.ts @@ -1048,20 +1048,26 @@ describe("byte-swapped universal Apple dispatch metadata", () => { }, ); - it("validates the selected thin slice's own byte order", () => { + it("reads the selected thin slice's CPU identity in its own byte order", () => { const bytes = wrapped(false); bytes.writeUInt32BE(0xfeedfacf, 256); + bytes.writeUInt32BE(0x0100000c, 260); + bytes.writeUInt32BE(0, 264); expect(() => decodeAppleDispatchMetadata(bytes, 100, provenance)).toThrow( - "Only little-endian 64-bit Mach-O metadata is supported", + "Unsupported big-endian Mach-O CPU type", ); }); it("rejects a 32-bit slice payload inside a little-endian container", () => { const bytes = wrapped(false); - bytes.writeUInt32BE(0xfeedface, 256); - expect(() => decodeAppleDispatchMetadata(bytes, 100, provenance)).toThrow( - "Unsupported 32-bit Mach-O metadata for requested CPU type arm64 (FAT header byte order: little-endian)", - ); + bytes.writeUInt32LE(12, 8); + bytes.writeUInt32LE(0, 12); + bytes.writeUInt32LE(0xfeedface, 256); + bytes.writeUInt32LE(12, 260); + bytes.writeUInt32LE(0, 264); + expect(() => + decodeAppleDispatchMetadata(bytes, 100, provenance, "arm"), + ).toThrow("Unsupported 32-bit Mach-O metadata for requested CPU type arm"); }); it("checks the full little-endian FAT64 slice extent", () => { diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index f5343bbfe2..a54658f3c0 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -35,7 +35,10 @@ import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; import { pushDispatchCoverage } from "./AppleDispatchCoverage.js"; import { parsePointerFixups } from "./AppleMachoFixups.js"; -import { parseMachoLayout } from "./AppleMachoSelection.js"; +import { + parseMachoLayout, + UnsupportedMachoMetadataFormatError, +} from "./AppleMachoSelection.js"; import { decodeObjcDispatchFacets } from "./AppleObjcDispatchFacets.js"; import { createSwiftRelativeReader, @@ -43,6 +46,7 @@ import { type DispatchRecordBudget, } from "./AppleSwiftDispatchFacets.js"; import { decodeSwiftClassVtables } from "./AppleSwiftVtables.js"; +import { inspectObjcClassImportBindings } from "./AppleObjcClassImportBindings.js"; /** Decode validated little-endian 64-bit Apple Objective-C metadata without loading target code. */ export const decodeAppleDispatchMetadata = ( @@ -50,6 +54,8 @@ export const decodeAppleDispatchMetadata = ( maxRecords: number, provenance: { path: string; sha256: string }, architecture = "arm64", + objcClassImportAddresses?: readonly string[], + objcClassImportAddressesComplete = true, ): ObjcSwiftMetadata => { const layout = parseMachoLayout(bytes, architecture); const { segments, sections, offset } = layout; @@ -59,6 +65,14 @@ export const decodeAppleDispatchMetadata = ( segments, layout.baseAddress, ); + const objcClassImports = inspectObjcClassImportBindings({ + bytes, + layout, + fixups, + addresses: objcClassImportAddresses, + addressesComplete: objcClassImportAddressesComplete, + maxRecords, + }); const u32 = (address: bigint) => bytes.readUInt32LE(offset(address, 4)); const decodePointer = (address: bigint) => fixups.decode(address, bytes.readBigUInt64LE(offset(address, 8))); @@ -96,7 +110,27 @@ export const decodeAppleDispatchMetadata = ( bytes.subarray(start, end), ); }; - const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const result = objcSwiftMetadataSchema.parse({ + db_save_result: null, + target_selection: { + kind: "macho-selected-image-v1", + container: layout.selectedImage.container, + offset: layout.selectedImage.offset, + byte_length: layout.selectedImage.byteLength, + cpu_type: layout.selectedImage.cpuType, + cpu_subtype: layout.selectedImage.cpuSubtype, + container_table_sha256: layout.selectedImage.containerTableSha256, + }, + ...(objcClassImportAddresses === undefined + ? {} + : { + objc_class_import_bindings: objcClassImports.bindings, + objc_class_import_binding_coverage: { + facet: "objc_class_import_bindings", + ...objcClassImports.coverage, + }, + }), + }); let records = 0; const budget: DispatchRecordBudget = { truncated: false, @@ -208,6 +242,8 @@ export const inspectAppleDispatchMetadata = async ( maxRecords: number, resources: ArtifactResourceScope, signal?: AbortSignal, + objcClassImportAddresses?: readonly string[], + objcClassImportAddressesComplete = true, ) => { if (target.kind !== "executable" || !isMachOFormat(target.format)) throw new TypeError("Apple dispatch metadata requires a Mach-O target"); @@ -257,6 +293,7 @@ export const inspectAppleDispatchMetadata = async ( ); return nativeDispatchMetadataResultSchema.parse({ target_sha256: target.sha256, + target_architecture: target.architecture, provider: APPLE_DISPATCH_METADATA_ANALYSIS_PROFILE.provider, analysis_profile_digest: APPLE_DISPATCH_METADATA_ANALYSIS_PROFILE.digest, result: decodeAppleDispatchMetadata( @@ -264,6 +301,8 @@ export const inspectAppleDispatchMetadata = async ( maxRecords, { path: target.path, sha256: digest }, target.architecture, + objcClassImportAddresses, + objcClassImportAddressesComplete, ), }); }; @@ -302,6 +341,13 @@ export const appleDispatchMetadataFailure = ( return failure; } if (cause instanceof AnalysisError) return cause; + if (cause instanceof UnsupportedMachoMetadataFormatError) + return new AnalysisCapabilityUnavailableError( + IDENTITY.id, + OPERATION, + cause.message, + { cause }, + ); if (cause instanceof ArtifactReaderFailure && cause.reason === "cancelled") return new AnalysisCancelledError(OPERATION, { cause: artifactFileCancellationCause(cause), diff --git a/src/native/AppleMachoFixups.test.ts b/src/native/AppleMachoFixups.test.ts index 4b6af1dd4a..8653b73e74 100644 --- a/src/native/AppleMachoFixups.test.ts +++ b/src/native/AppleMachoFixups.test.ts @@ -11,17 +11,21 @@ const BASE = 0x100000000n; const IMPORTS = [ { symbol: "_OBJC_CLASS_$_NSObject", + binding_source: "chained", + library_ordinal: 1, library: "/usr/lib/libobjc.A.dylib", weak: false, addend: 0n, }, { symbol: "_free", + binding_source: "chained", + library_ordinal: 2, library: "/usr/lib/libSystem.B.dylib", weak: true, addend: 4n, }, -]; +] as const; const field = (value: bigint, from: number): bigint => value << BigInt(from); @@ -40,6 +44,8 @@ describe("chained pointer decoding", () => { expect(decodeChainedPointer(bind, 6, BASE, IMPORTS)).toEqual({ kind: "bind", symbol: "_free", + binding_source: "chained", + library_ordinal: 2, library: "/usr/lib/libSystem.B.dylib", weak: true, addend: 6n, @@ -190,7 +196,7 @@ describe("independent chained segment coverage", () => { chained: { offset: 0, size: imports }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, segments, BASE, @@ -217,7 +223,7 @@ describe("pointer fixup tables", () => { chained: { offset: 0, size: data.length }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -233,7 +239,7 @@ describe("pointer fixup tables", () => { chained: { offset: 0, size: data.length }, binds: [], rebases: [{ offset: 8, size: 12 }], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -255,7 +261,7 @@ describe("pointer fixup tables", () => { chained: { offset: 0, size: 128 }, binds: [], rebases: [{ offset: 8, size: 12 }], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -330,7 +336,7 @@ describe("parsed chained pointer membership", () => { chained: { offset: 0, size: 128 }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -422,7 +428,7 @@ describe("parsed chained pointer membership", () => { chained: { offset: 0, size: imports }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, segments, BASE, @@ -479,7 +485,7 @@ const aliasedStarts = (format: number, count = 32, pages = 128) => { chained: { offset: 0, size: end }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, }; }; @@ -609,7 +615,7 @@ describe("chained pointer membership", () => { chained: { offset: 0, size: 160 }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -639,7 +645,7 @@ describe("chained pointer membership", () => { chained: { offset: 0, size: 128 }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -675,7 +681,7 @@ describe("chained pointer membership", () => { chained: { offset: 0, size: 128 }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], } as const; const fixups = parsePointerFixups(data, commands, SEGMENTS, BASE); expect(fixups.decode(BASE + 0x4008n, 0n)).toEqual({ @@ -749,7 +755,7 @@ describe("malformed chained pointer coverage", () => { chained: { offset: 0, size: 128 }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -785,7 +791,7 @@ describe("malformed chained pointer coverage", () => { chained: { offset: 0, size: 128 }, binds: [], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -796,32 +802,26 @@ describe("malformed chained pointer coverage", () => { }); describe("chained import library identity", () => { it.each([1, 2, 3] as const)( - "retains positive and special library ordinals for import format %i", + "retains valid positive and special ordinals and rejects invalid ones for format %i", (format) => { const data = chainedData(format); const importAt = data.readUInt32LE(8); - const width = format === 3 ? 16 : 8; - const maximum = format === 3 ? 0xfff0 : 0xf0; - const libraries = Array.from( - { length: maximum }, - (_, index) => `/library-${index + 1}`, - ); - for (const ordinal of [ - 1 << (width - 1), - maximum, - maximum + 1, - (1 << width) - 1, - ]) { + const libraries = ["/library-1"]; + const writeOrdinal = (ordinal: number) => { if (format === 3) data.writeBigUInt64LE( - (data.readBigUInt64LE(importAt) & ~0xffffn) | BigInt(ordinal), + (data.readBigUInt64LE(importAt) & ~0xffffn) | + BigInt(ordinal & 0xffff), importAt, ); else data.writeUInt32LE( - (data.readUInt32LE(importAt) & ~0xff) | ordinal, + (data.readUInt32LE(importAt) & ~0xff) | (ordinal & 0xff), importAt, ); + }; + for (const ordinal of [1, 0, -1, -2, -3]) { + writeOrdinal(ordinal); data.writeBigUInt64LE(field(1n, 63), 0x4008); const fixups = parsePointerFixups( data, @@ -838,9 +838,46 @@ describe("chained import library identity", () => { fixups.decode(BASE + 0x4008n, data.readBigUInt64LE(0x4008)), ).toMatchObject({ kind: "bind", - library: ordinal <= maximum ? libraries[ordinal - 1] : null, + library_ordinal: ordinal, + library: ordinal === 1 ? libraries[0] : null, }); } + writeOrdinal(2); + const invalidPositive = parsePointerFixups( + data, + { + chained: { offset: 0, size: 160 }, + binds: [], + rebases: [], + dylibs: libraries, + }, + SEGMENTS, + BASE, + ); + expect( + invalidPositive.decode(BASE + 0x4008n, data.readBigUInt64LE(0x4008)), + ).toMatchObject({ + kind: "unsupported", + reason: "Dylib ordinal 2 exceeds the 1 declared dependencies", + }); + writeOrdinal(-4); + const invalidSpecial = parsePointerFixups( + data, + { + chained: { offset: 0, size: 160 }, + binds: [], + rebases: [], + dylibs: libraries, + }, + SEGMENTS, + BASE, + ); + expect( + invalidSpecial.decode(BASE + 0x4008n, data.readBigUInt64LE(0x4008)), + ).toMatchObject({ + kind: "unsupported", + reason: "Unsupported dylib ordinal -4", + }); }, ); }); @@ -948,6 +985,34 @@ describe("legacy pointer fixup streams", () => { target: 0x99n, }); }); + + it("retains weak-bind stream provenance independently from its default ordinal", () => { + const opcodes = Buffer.from([ + 0x40, + ...Buffer.from("_OBJC_CLASS_$_NSObject\0"), + 0x71, + 0x18, + 0x90, + 0x00, + ]); + const fixups = parsePointerFixups( + opcodes, + { + chained: null, + binds: [{ offset: 0, size: opcodes.length, stream: "weak" }], + rebases: [], + dylibs: [], + }, + SEGMENTS, + BASE, + ); + expect(fixups.decode(BASE + 0x4018n, 0n)).toMatchObject({ + kind: "bind", + binding_source: "weak", + library_ordinal: 0, + library: null, + }); + }); }); describe("legacy bind operand decoding", () => { @@ -1045,7 +1110,7 @@ describe("legacy bind operand decoding", () => { chained: null, binds: [{ offset: 0, size: opcodes.length - 1, stream: "bind" }], rebases: [], - dylibs: [], + dylibs: ["/library-1", "/library-2"], }, SEGMENTS, BASE, @@ -1149,3 +1214,81 @@ describe("Objective-C property attributes", () => { }); }); }); + +it("preserves classic special ordinals and returns unsupported at invalid bind locations", () => { + const parseOrdinal = (ordinal: number, dylibs: readonly string[]) => { + const immediate = ordinal < 0 ? ordinal + 16 : ordinal; + const opcodes = Buffer.from([ + ordinal > 0 ? 0x10 | ordinal : 0x30 | immediate, + 0x40, + ...Buffer.from("_OBJC_CLASS_$_NSObject\0"), + 0x71, + 0x08, + 0x90, + 0x00, + ]); + return parsePointerFixups( + opcodes, + { + chained: null, + binds: [{ offset: 0, size: opcodes.length, stream: "bind" }], + rebases: [], + dylibs, + }, + SEGMENTS, + BASE, + ); + }; + for (const ordinal of [0, -1, -2, -3]) { + const fixups = parseOrdinal(ordinal, []); + expect(fixups.decode(BASE + 0x4008n, 0n)).toMatchObject({ + kind: "bind", + symbol: "_OBJC_CLASS_$_NSObject", + library_ordinal: ordinal, + library: null, + }); + } + for (const [ordinal, dylibs, reason] of [ + [1, [], "Dylib ordinal 1 exceeds the 0 declared dependencies"], + [-4, [], "Unsupported dylib ordinal -4"], + ] as const) { + const fixups = parseOrdinal(ordinal, dylibs); + expect(fixups.decode(BASE + 0x4008n, 0n)).toEqual({ + kind: "unsupported", + reason, + }); + } + const oversized = BigInt(Number.MAX_SAFE_INTEGER) + 1n; + let remaining = oversized; + const encoded: number[] = []; + while (remaining >= 0x80n) { + encoded.push(Number(remaining & 0x7fn) | 0x80); + remaining >>= 7n; + } + encoded.push(Number(remaining)); + const oversizedStream = Buffer.from([ + 0x20, + ...encoded, + 0x40, + ...Buffer.from("_OBJC_CLASS_$_NSObject\0"), + 0x71, + 0x08, + 0x90, + 0x00, + ]); + const oversizedFixups = parsePointerFixups( + oversizedStream, + { + chained: null, + binds: [{ offset: 0, size: oversizedStream.length, stream: "bind" }], + rebases: [], + dylibs: [], + }, + SEGMENTS, + BASE, + ); + expect(oversizedFixups.decode(BASE + 0x4008n, 0n)).toEqual({ + kind: "unsupported", + reason: `Dylib ordinal ${oversized} exceeds the safe integer range`, + }); +}); diff --git a/src/native/AppleMachoFixups.ts b/src/native/AppleMachoFixups.ts index f7eaa9fb25..50bb241477 100644 --- a/src/native/AppleMachoFixups.ts +++ b/src/native/AppleMachoFixups.ts @@ -27,6 +27,10 @@ export type DecodedPointer = | { readonly kind: "bind"; readonly symbol: string; + /** Producer stream, which is part of lookup identity for weak binds. */ + readonly binding_source: "chained" | "bind" | "weak" | "lazy"; + /** Signed dyld library ordinal; 0 and negative values are special scopes. */ + readonly library_ordinal: number; readonly library: string | null; readonly weak: boolean; readonly addend: bigint; @@ -66,6 +70,8 @@ const signed = (value: bigint, width: number): bigint => interface Import { readonly symbol: string; + readonly binding_source: "chained" | "bind" | "weak" | "lazy"; + readonly library_ordinal: number; readonly library: string | null; readonly weak: boolean; readonly addend: bigint; @@ -75,15 +81,26 @@ interface BindRun { readonly start: bigint; count: bigint; readonly stride: bigint; - readonly binding: Import; + readonly binding: + | { readonly kind: "bind"; readonly value: Import } + | { readonly kind: "unsupported"; readonly reason: string }; } -/** Library ordinal to install name; 0 and negative ordinals are special lookups. */ +/** Resolve only valid dylib ordinals; preserve special scopes as raw ordinals. */ const libraryName = ( ordinal: number, dylibs: readonly string[], ): string | null => { - if (ordinal > 0) return dylibs[ordinal - 1] ?? null; + if (!Number.isSafeInteger(ordinal) || ordinal < -3) + throw new RangeError(`Unsupported dylib ordinal ${String(ordinal)}`); + if (ordinal > 0) { + const name = dylibs[ordinal - 1]; + if (name === undefined) + throw new RangeError( + `Dylib ordinal ${ordinal} exceeds the ${dylibs.length} declared dependencies`, + ); + return name; + } return null; }; @@ -132,6 +149,8 @@ const readImport = ( base + header.symbols + Number(bits(word, 32, 32)), end, ), + binding_source: "chained", + library_ordinal: ordinal, library: libraryName(ordinal, dylibs), weak: bits(word, 16, 1) === 1n, addend: bytes.readBigInt64LE(at + 8), @@ -143,6 +162,8 @@ const readImport = ( encodedOrdinal > 0xf0 ? encodedOrdinal - 0x100 : encodedOrdinal; return { symbol: cString(bytes, base + header.symbols + (word >>> 9), end), + binding_source: "chained", + library_ordinal: ordinal, library: libraryName(ordinal, dylibs), weak: ((word >>> 8) & 1) === 1, addend: header.format === 2 ? BigInt(bytes.readInt32LE(at + 4)) : 0n, @@ -549,6 +570,7 @@ const sleb = (bytes: Buffer, cursor: { at: number }, end: number): bigint => { /** Registers of the `LC_DYLD_INFO` bind state machine. */ interface BindState { ordinal: number; + ordinalFailure: string | null; segmentIndex: number; symbol: string; weak: boolean; @@ -569,12 +591,23 @@ const setBindRegister = ( switch (byte & 0xf0) { case 0x10: state.ordinal = immediate; + state.ordinalFailure = null; return true; case 0x20: - state.ordinal = Number(uleb(bytes, cursor, end)); + { + const ordinal = uleb(bytes, cursor, end); + if (ordinal > BigInt(Number.MAX_SAFE_INTEGER)) { + state.ordinal = 0; + state.ordinalFailure = `Dylib ordinal ${ordinal} exceeds the safe integer range`; + } else { + state.ordinal = Number(ordinal); + state.ordinalFailure = null; + } + } return true; case 0x30: state.ordinal = immediate === 0 ? 0 : immediate - 16; + state.ordinalFailure = null; return true; case 0x40: { const stop = bytes.indexOf(0, cursor.at); @@ -621,6 +654,7 @@ const readBindStream = ( const cursor = { at: range.offset }; const state: BindState = { ordinal: 0, + ordinalFailure: null, segmentIndex: -1, symbol: "", weak: false, @@ -641,24 +675,48 @@ const readBindStream = ( if (count === 0n) return; preflightRun(count, skip); const stride = 8n + skip; - const binding: Import = { - symbol: state.symbol, - library: - range.stream === "weak" - ? null - : libraryName(state.ordinal, image.dylibs), - weak: state.weak, - addend: state.addend, - }; + let binding: BindRun["binding"]; + try { + if (state.ordinalFailure !== null) + throw new RangeError(state.ordinalFailure); + const resolvedLibrary = libraryName(state.ordinal, image.dylibs); + binding = { + kind: "bind", + value: { + symbol: state.symbol, + binding_source: range.stream, + library_ordinal: state.ordinal, + library: range.stream === "weak" ? null : resolvedLibrary, + weak: state.weak, + addend: state.addend, + }, + }; + } catch (cause: unknown) { + binding = { + kind: "unsupported", + reason: cause instanceof Error ? cause.message : String(cause), + }; + } const previous = binds.at(-1); + const sameBinding = ( + left: BindRun["binding"], + right: BindRun["binding"], + ) => + left.kind === "unsupported" && right.kind === "unsupported" + ? left.reason === right.reason + : left.kind === "bind" && right.kind === "bind" + ? left.value.symbol === right.value.symbol && + left.value.binding_source === right.value.binding_source && + left.value.library_ordinal === right.value.library_ordinal && + left.value.library === right.value.library && + left.value.weak === right.value.weak && + left.value.addend === right.value.addend + : false; if ( previous !== undefined && previous.stride === stride && previous.start + previous.count * stride === state.address && - previous.binding.symbol === binding.symbol && - previous.binding.library === binding.library && - previous.binding.weak === binding.weak && - previous.binding.addend === binding.addend + sameBinding(previous.binding, binding) ) { previous.count += count; } else { @@ -769,7 +827,9 @@ export const parsePointerFixups = ( } return match === undefined ? { kind: "rebase", target: raw } - : { kind: "bind", ...match.binding }; + : match.binding.kind === "unsupported" + ? match.binding + : { kind: "bind", ...match.binding.value }; }, }; }; diff --git a/src/native/AppleMachoSelection.test.ts b/src/native/AppleMachoSelection.test.ts index 81c34fa7b1..08fb8630bc 100644 --- a/src/native/AppleMachoSelection.test.ts +++ b/src/native/AppleMachoSelection.test.ts @@ -8,13 +8,33 @@ import { machoImage, machoImageWithUnknownCommand, } from "../artifacts/apple/MachoImage.fixture.js"; -import { parseMachoLayout, selectMachoSlice } from "./AppleMachoSelection.js"; +import { + parseMachoLayout, + selectMachoSlice, + UnsupportedMachoMetadataFormatError, +} from "./AppleMachoSelection.js"; import { decodeAppleDispatchMetadata } from "./AppleDispatchMetadata.js"; /** macOS 27 system binaries ship CPU_SUBTYPE_ARM64E_X1 (12) with the PAC ABI bit. */ const ARM64E_X1 = { type: 0x0100000c, subtype: 0x8000000c } as const; const X86_64H = { type: 0x01000007, subtype: 8 } as const; +it("keeps unknown magic distinct from recognized unsupported Mach-O formats", () => { + expect(() => parseMachoLayout(Buffer.alloc(10), "arm64")).toThrow( + "Truncated Mach-O header", + ); + + const unknown = Buffer.alloc(32); + let failure: unknown; + try { + parseMachoLayout(unknown, "arm64"); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(TypeError); + expect(failure).not.toBeInstanceOf(UnsupportedMachoMetadataFormatError); +}); + describe("native Mach-O slice selection", () => { it("rejects misaligned load commands", () => { expect(() => @@ -56,6 +76,43 @@ describe("native Mach-O slice selection", () => { expect(parseMachoLayout(Buffer.from(fat), "arm64").segments).toEqual([]); }); + it("retains the selected image range and complete raw CPU identity", () => { + const thin = Buffer.from(machoImage({ cpu: CPU.arm64e })); + expect(parseMachoLayout(thin, "arm64").selectedImage).toEqual({ + container: "thin", + offset: 0, + byteLength: thin.length, + cpuType: CPU.arm64e.type, + cpuSubtype: CPU.arm64e.subtype, + containerTableSha256: null, + }); + + const fat = Buffer.from( + fatImage([{ cpu: ARM64E_X1, bytes: machoImage({ cpu: ARM64E_X1 }) }]), + ); + const selection = parseMachoLayout(fat, "arm64").selectedImage; + expect(selection).toMatchObject({ + container: "fat", + cpuType: ARM64E_X1.type, + cpuSubtype: ARM64E_X1.subtype, + }); + expect(selection.offset).toBeGreaterThan(0); + expect(selection.byteLength).toBeGreaterThan(0); + expect(selection.containerTableSha256).toMatch(/^[a-f0-9]{64}$/u); + }); + + it("rejects a FAT 32-bit declaration whose embedded CPU identity disagrees", () => { + const fat = fatImage([ + { cpu: CPU.i386, bytes: machoImage({ cpu: CPU.i386, wide: false }) }, + ]); + const sliceOffset = new DataView(fat.buffer).getUint32(16, false); + new DataView(fat.buffer).setUint32(sliceOffset + 4, CPU.arm64.type, true); + + expect(() => parseMachoLayout(Buffer.from(fat), "x86")).toThrow( + /FAT architecture 0 CPU type\/subtype disagrees with its Mach-O header/u, + ); + }); + it("selects only the admitted CPU type and rejects a 32-bit slice", () => { const sixtyFour = fatImage([ { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, diff --git a/src/native/AppleMachoSelection.ts b/src/native/AppleMachoSelection.ts index bcd07cc6c3..ee0bae2b18 100644 --- a/src/native/AppleMachoSelection.ts +++ b/src/native/AppleMachoSelection.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import type { FixupCommands } from "./AppleMachoFixups.js"; import { MACHO_ARCHITECTURE_POLICIES, @@ -46,6 +48,15 @@ export interface MachoSlice { export interface MachoLayout { segments: Segment[]; sections: Section[]; + /** CPU identity and exact byte range selected from this source artifact. */ + selectedImage: { + readonly container: "thin" | "fat"; + readonly offset: number; + readonly byteLength: number; + readonly cpuType: number; + readonly cpuSubtype: number; + readonly containerTableSha256: string | null; + }; /** Fixup load commands and dependent dylibs of the selected slice. */ fixups: FixupCommands; /** VM address of the segment that maps file offset 0 (the preferred load address). */ @@ -135,18 +146,20 @@ export const selectMachoSlice = ( sliceEnd: best.offset + best.size, fatDeclaration: { ...best, tableEnd: headerEnd }, }; - if (selected.slice + 4 > bytes.length) - throw malformed("Truncated Mach-O header"); - const sliceMagic = bytes.readUInt32LE(selected.slice); - if (sliceMagic === MH_MAGIC || sliceMagic === MH_CIGAM) - throw invalid( - `Unsupported 32-bit Mach-O metadata for requested CPU type ${architecture}`, - ); return selected; }; const MH_MAGIC = 0xfeedface; const MH_CIGAM = 0xcefaedfe; +const MH_CIGAM_64 = 0xcffaedfe; + +/** A recognized Mach-O class outside this metadata reader's supported formats. */ +export class UnsupportedMachoMetadataFormatError extends Error { + constructor(message: string) { + super(message); + this.name = "UnsupportedMachoMetadataFormatError"; + } +} /** Parse segments/sections for a little-endian 64-bit Mach-O architecture slice. */ export const parseMachoLayout = ( @@ -156,20 +169,51 @@ export const parseMachoLayout = ( if (bytes.length < 4) throw new RangeError("Truncated Mach-O header"); const selection = selectMachoSlice(bytes, architecture); const { slice, sliceEnd } = selection; - if (slice === 0 && sliceEnd === bytes.length && bytes.length < 32) + if (slice + 4 > sliceEnd) throw new RangeError("Truncated Mach-O header"); + const selectedMagic = bytes.readUInt32LE(slice); + const is32Bit = selectedMagic === MH_MAGIC || selectedMagic === MH_CIGAM; + const isBigEndian = + selectedMagic === MH_CIGAM || selectedMagic === MH_CIGAM_64; + const headerSize = is32Bit ? 28 : 32; + const recognizedMagic = + is32Bit || selectedMagic === 0xfeedfacf || selectedMagic === MH_CIGAM_64; + if (!recognizedMagic && slice + 32 > sliceEnd) throw new RangeError("Truncated Mach-O header"); - if (slice + 32 > sliceEnd || bytes.readUInt32LE(slice) !== 0xfeedfacf) + if (!recognizedMagic) throw new TypeError( "Only little-endian 64-bit Mach-O metadata is supported", ); + if (slice + headerSize > sliceEnd) + throw new RangeError("Truncated Mach-O header"); + const readHeaderUInt32 = isBigEndian + ? (offset: number) => bytes.readUInt32BE(offset) + : (offset: number) => bytes.readUInt32LE(offset); + const cpuType = readHeaderUInt32(slice + 4); + const cpuSubtype = readHeaderUInt32(slice + 8); const fat = selection.fatDeclaration; if (fat !== undefined) { const issue = validateFatSlice(fat, bytes.length, fat.tableEnd, { - cpuType: bytes.readUInt32LE(slice + 4), - cpuSubtype: bytes.readUInt32LE(slice + 8), + cpuType, + cpuSubtype, }); if (issue !== null) throw new TypeError(issue); } + if (is32Bit) { + if (MACHO_ARCHITECTURE_POLICIES[architecture]?.cpuType !== cpuType) + throw new TypeError( + `Mach-O CPU type does not match requested architecture ${architecture}`, + ); + throw new UnsupportedMachoMetadataFormatError( + `Unsupported 32-bit Mach-O metadata for requested CPU type ${architecture}`, + ); + } + if (selectedMagic === MH_CIGAM_64) { + if (cpuType !== 0x01000012) + throw new TypeError("Unsupported big-endian Mach-O CPU type"); + throw new UnsupportedMachoMetadataFormatError( + "Unsupported big-endian 64-bit PowerPC Mach-O metadata", + ); + } const commands = bytes.readUInt32LE(slice + 16); const commandEnd = slice + 32 + bytes.readUInt32LE(slice + 20); if (commands > 4096 || commandEnd > sliceEnd) @@ -221,6 +265,19 @@ export const parseMachoLayout = ( return { segments, sections, + selectedImage: { + container: fat === undefined ? "thin" : "fat", + offset: slice, + byteLength: sliceEnd - slice, + cpuType, + cpuSubtype, + containerTableSha256: + fat === undefined + ? null + : createHash("sha256") + .update(bytes.subarray(0, fat.tableEnd)) + .digest("hex"), + }, offset, fixups: { chained, binds, rebases, dylibs }, baseAddress: state.baseAddress ?? 0n, diff --git a/src/native/AppleObjcClassImportBindings.test.ts b/src/native/AppleObjcClassImportBindings.test.ts new file mode 100644 index 0000000000..89cf06bcfa --- /dev/null +++ b/src/native/AppleObjcClassImportBindings.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it } from "vitest"; + +import type { PointerFixups } from "./AppleMachoFixups.js"; +import type { MachoLayout } from "./AppleMachoSelection.js"; +import { inspectObjcClassImportBindings } from "./AppleObjcClassImportBindings.js"; + +const createInput = ( + overrides: Partial[0]> = {}, +) => { + const bytes = Buffer.alloc(16); + const fixups: PointerFixups = { + kind: "chained", + formats: ["DYLD_CHAINED_PTR_64"], + failures: [], + issues: [], + examined: 1, + decoded: 1, + unreadRebaseBytes: 0, + decode: (address) => + address === 0x1000n + ? { + kind: "bind", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: false, + addend: 0n, + } + : address === 0x1008n + ? { + kind: "bind", + symbol: "_free", + binding_source: "chained", + library_ordinal: 2, + library: "/usr/lib/libSystem.B.dylib", + weak: false, + addend: 0n, + } + : { kind: "rebase", target: 0n }, + }; + const layout: MachoLayout = { + segments: [], + sections: [], + selectedImage: { + container: "thin", + offset: 0, + byteLength: bytes.length, + cpuType: 0x0100000c, + cpuSubtype: 0, + containerTableSha256: null, + }, + fixups: { chained: null, binds: [], rebases: [], dylibs: [] }, + baseAddress: 0n, + offset: (address, size = 1) => { + if (address === 0x1000n && size === 8) return 0; + if (address === 0x1008n && size === 8) return 8; + throw new RangeError("address is outside fixture"); + }, + }; + return { + bytes, + layout, + fixups, + addresses: ["0x1000", "0x1008"], + addressesComplete: true, + maxRecords: 100, + ...overrides, + }; +}; + +describe("Objective-C import binding address inspection", () => { + it("retains only exact class-symbol binds for the requested slot addresses", () => { + expect(inspectObjcClassImportBindings(createInput())).toEqual({ + bindings: [ + { + requested_address: "0x1000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: false, + addend: "0", + }, + ], + coverage: { + status: "complete", + reason: null, + examined: 2, + decoded: 1, + }, + }); + }); + + it("keeps exact binds while marking malformed or unresolved addresses partial", () => { + expect( + inspectObjcClassImportBindings( + createInput({ addresses: ["0x1000", "not-an-address", "0x2000"] }), + ), + ).toMatchObject({ + bindings: [{ requested_address: "0x1000" }], + coverage: { + status: "partial", + examined: 3, + decoded: 1, + reason: expect.stringContaining("not a 64-bit hexadecimal address"), + }, + }); + }); + + it("keeps an invalid raw ordinal unsupported instead of treating it as an absent bind", () => { + const input = createInput(); + expect( + inspectObjcClassImportBindings({ + ...input, + addresses: ["0x1000"], + fixups: { + ...input.fixups, + decode: () => ({ + kind: "unsupported", + reason: "Dylib ordinal 4 exceeds the 3 declared dependencies", + }), + }, + }), + ).toEqual({ + bindings: [], + coverage: { + status: "partial", + reason: "0x1000: Dylib ordinal 4 exceeds the 3 declared dependencies", + examined: 1, + decoded: 0, + }, + }); + }); + + it("does not infer absent imports from unsupported fixup coverage", () => { + const input = createInput({ + fixups: { ...createInput().fixups, kind: "none" }, + addresses: ["0x1000"], + }); + expect(inspectObjcClassImportBindings(input)).toMatchObject({ + bindings: [], + coverage: { + status: "unsupported", + reason: expect.stringContaining("no supported pointer fixup commands"), + }, + }); + }); + + it("preserves partial facts when the existing record budget truncates candidates", () => { + expect( + inspectObjcClassImportBindings( + createInput({ + addresses: ["0x1000", "0x1008"], + maxRecords: 1, + }), + ), + ).toMatchObject({ + bindings: [{ requested_address: "0x1000" }], + coverage: { + status: "partial", + examined: 1, + decoded: 1, + reason: expect.stringContaining("max_records limit"), + }, + }); + }); +}); diff --git a/src/native/AppleObjcClassImportBindings.ts b/src/native/AppleObjcClassImportBindings.ts new file mode 100644 index 0000000000..d6fa3fcdce --- /dev/null +++ b/src/native/AppleObjcClassImportBindings.ts @@ -0,0 +1,119 @@ +import type { PointerFixups } from "./AppleMachoFixups.js"; +import type { MachoLayout } from "./AppleMachoSelection.js"; +import type { ObjcClassImportBinding } from "../domain/native/objcSwiftMetadata.js"; + +export interface ObjcClassImportBindingScan { + readonly bindings: readonly ObjcClassImportBinding[]; + readonly coverage: { + readonly status: "complete" | "partial" | "unsupported" | "not_requested"; + readonly reason: string | null; + readonly examined: number; + readonly decoded: number; + }; +} + +/** Resolve caller-supplied names only at their exact selected-image addresses. */ +export const inspectObjcClassImportBindings = (input: { + readonly bytes: Buffer; + readonly layout: MachoLayout; + readonly fixups: PointerFixups; + readonly addresses: readonly string[] | undefined; + readonly addressesComplete: boolean; + readonly maxRecords: number; +}): ObjcClassImportBindingScan => { + const { bytes, layout, fixups, addresses, addressesComplete, maxRecords } = + input; + if (addresses === undefined || addresses.length === 0) + return { + bindings: [], + coverage: { + status: "not_requested", + reason: null, + examined: 0, + decoded: 0, + }, + }; + if (fixups.kind === "none") + return { + bindings: [], + coverage: { + status: "unsupported", + reason: "Selected Mach-O contains no supported pointer fixup commands", + examined: 0, + decoded: 0, + }, + }; + + const reasons = new Set(); + if (!addressesComplete) + reasons.add( + "Candidate address scan reached the existing max_records limit", + ); + for (const failure of fixups.failures) reasons.add(failure); + + const bindings: ObjcClassImportBinding[] = []; + const examinedAddresses = new Set(); + const candidates = addresses.slice(0, maxRecords); + if (candidates.length < addresses.length) + reasons.add( + "Candidate address scan reached the existing max_records limit", + ); + + for (const requestedAddress of candidates) { + if (examinedAddresses.has(requestedAddress)) continue; + examinedAddresses.add(requestedAddress); + const parsedAddress = parseAddress(requestedAddress); + if (parsedAddress === undefined) { + reasons.add("A provider address is not a 64-bit hexadecimal address"); + continue; + } + + try { + const offset = layout.offset(parsedAddress, 8); + const raw = bytes.readBigUInt64LE(offset); + const decoded = fixups.decode(parsedAddress, raw); + if (decoded.kind === "unsupported") { + reasons.add(`${requestedAddress}: ${decoded.reason}`); + continue; + } + if (decoded.kind !== "bind" || !isObjcClassImport(decoded.symbol)) + continue; + bindings.push({ + requested_address: requestedAddress, + symbol: decoded.symbol, + binding_source: decoded.binding_source, + library_ordinal: decoded.library_ordinal, + library: decoded.library, + weak: decoded.weak, + addend: decoded.addend.toString(10), + }); + } catch (cause: unknown) { + reasons.add( + `${requestedAddress}: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + } + + const reason = [...reasons].join("; ") || null; + return { + bindings, + coverage: { + status: reason === null ? "complete" : "partial", + reason, + examined: examinedAddresses.size, + decoded: bindings.length, + }, + }; +}; + +const parseAddress = (address: string): bigint | undefined => { + if (!/^0x[0-9a-f]{1,16}$/iu.test(address)) return undefined; + try { + return BigInt(address); + } catch { + return undefined; + } +}; + +const isObjcClassImport = (symbol: string): boolean => + /^_?OBJC_CLASS_\$_.+$/u.test(symbol); diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index f3e8da8e08..e48422cc7e 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -278,6 +278,8 @@ class NativeMacOSClient implements AnalysisClient { input.data.max_records, this.#dispatchResources, options?.signal, + input.data.objc_class_import_addresses, + input.data.objc_class_import_addresses_complete ?? true, ), ); return ok( diff --git a/tests/acceptance/nativeDispatchSnapshot.test.ts b/tests/acceptance/nativeDispatchSnapshot.test.ts index cb2df2fafe..17b81dd1b7 100644 --- a/tests/acceptance/nativeDispatchSnapshot.test.ts +++ b/tests/acceptance/nativeDispatchSnapshot.test.ts @@ -13,6 +13,7 @@ import { import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +import { nativeObjcClassImportMachO } from "../fixtures/nativeObjcClassImportMachO.js"; cliTest.skipIf(process.platform !== "darwin")( "retains and reopens profileless native dispatch workflow snapshots through CLI and stdio MCP", @@ -116,3 +117,80 @@ cliTest.skipIf(process.platform !== "darwin")( }, 30000, ); + +cliTest.skipIf(process.platform !== "darwin")( + "reports exact Objective-C class imports through the compiled native MCP adapter", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-objc-import-mcp-"); + const target = join(root, "class-import.macho"); + const fixture = nativeObjcClassImportMachO(); + await writeFile(target, fixture.bytes); + const environment = { + PATH: process.env.PATH ?? "", + REA_ANALYSIS_PROVIDER: "auto", + HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", + GHIDRA_INSTALL_DIR: "/rea-unconfigured-deep-provider/ghidra", + JAVA_HOME: "/rea-unconfigured-deep-provider/java", + REA_IDA_MCP_CONFIG: "/rea-unconfigured-deep-provider/ida-mcp.json", + REA_KUNA_COMMAND: "/rea-unconfigured-deep-provider/kuna", + REA_KUNA_CACHE_DIR: "/rea-unconfigured-deep-provider/kuna-cache", + REA_LOG_LEVEL: "silent", + }; + + const cliResult = await cli.run({ + arguments: ["inspect-native-dispatch-metadata", target, "--json"], + environment, + }); + expect(cliResult.exitCode, cliResult.stderr).toBe(0); + const cliEvidence = toolContract( + "inspect_native_dispatch_metadata", + ).outputSchema.parse(cliResult.json); + expect(cliEvidence.normalized_result).toMatchObject({ + result: { + target_selection: { + container: "thin", + offset: 0, + cpu_type: CPU.arm64.type, + cpu_subtype: CPU.arm64.subtype, + }, + }, + }); + + const { client, call } = await connectLocalToolsMcp(environment); + const opened = await call("open_binary", { path: target }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspected = await call("inspect_native_dispatch_metadata", { + max_records: 100, + objc_class_import_addresses: [fixture.address, fixture.unboundAddress], + objc_class_import_addresses_complete: true, + }); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + const evidence = parseEvidence(inspected.structuredContent); + expect(evidence.normalized_result).toMatchObject({ + target_sha256: expect.stringMatching(/^[a-f0-9]{64}$/u), + target_architecture: "arm64", + provider: { id: "native-macos" }, + result: { + target_selection: { + container: "thin", + offset: 0, + cpu_type: CPU.arm64.type, + cpu_subtype: CPU.arm64.subtype, + }, + objc_class_import_bindings: [ + { + requested_address: fixture.address, + symbol: "_OBJC_CLASS_$_Document", + library: "/System/Library/Frameworks/AppKit.framework/AppKit", + }, + ], + objc_class_import_binding_coverage: { + status: "complete", + examined: 2, + decoded: 1, + }, + }, + }); + await client.ping(); + }, +); diff --git a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts index 35240f1158..c4a0d6b233 100644 --- a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts +++ b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts @@ -10,12 +10,19 @@ import type { AnalysisExecution, AnalysisOperationPort, } from "../../../src/application/AnalysisProvider.js"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; +import type { AnalysisOperation } from "../../../src/application/AnalysisProvider.js"; import type { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; import { AnalysisCapabilityUnavailableError, AnalysisProtocolError, } from "../../../src/domain/analysisErrorCore.js"; -import { err, type Result } from "../../../src/domain/result.js"; +import { + err, + ok as resultOk, + type Result, +} from "../../../src/domain/result.js"; +import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; import { observed as ok } from "../../fixtures/analysisExecution.js"; afterEach(closeEnhancedToolResources); @@ -37,9 +44,19 @@ const connect = (analysis: AnalysisOperationPort) => : analysis.execute(operation, parameters, options), }); -it("discovers declarations without counting duplicates or compiler bookkeeping", async () => { +it("retains distinct class locations and excludes compiler bookkeeping", async () => { const client = await connect({ execute: (name) => { + if (name === "inspect_native_dispatch_metadata") + return Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + "fixture", + name, + "Native metadata is unavailable in this provider fixture", + ), + ), + ); expect(name).toBe("list_names"); return Promise.resolve( ok([ @@ -68,12 +85,14 @@ it("discovers declarations without counting duplicates or compiler bookkeeping", await client.callTool({ name: "get_objc_classes", arguments: {} }), ); expect(result).toMatchObject({ - count: 3, + count: 4, classes: [ { address: "0x1", name: "_OBJC_CLASS_$_First" }, { address: "0x2", name: "_OBJC_CLASS_$_Last" }, + { address: "0x3", name: "_OBJC_CLASS_$_First" }, { address: "0x10", name: "l_OBJC_CLASS_Legacy" }, ], + identity_coverage: { status: "unknown" }, }); expect( jsonResult( @@ -83,8 +102,12 @@ it("discovers declarations without counting duplicates or compiler bookkeeping", }), ), ).toMatchObject({ - count: 1, - classes: [{ address: "0x1", name: "_OBJC_CLASS_$_First" }], + count: 2, + identity_coverage: { status: "unknown" }, + classes: [ + { address: "0x1", name: "_OBJC_CLASS_$_First" }, + { address: "0x3", name: "_OBJC_CLASS_$_First" }, + ], }); expect( jsonResult( @@ -112,6 +135,16 @@ it("discovers stripped Hopper class labels through MCP with literal filtering", ]; const client = await connect({ execute: (name) => { + if (name === "inspect_native_dispatch_metadata") + return Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + "fixture", + name, + "Native metadata is unavailable in this provider fixture", + ), + ), + ); expect(name).toBe("list_names"); return Promise.resolve( ok( @@ -132,6 +165,7 @@ it("discovers stripped Hopper class labels through MCP with literal filtering", ); expect(all).toMatchObject({ count: 5, + identity_coverage: { status: "unknown" }, classes: names.slice(0, 5).map((name, index) => ({ address: `0x${index + 1}`, name, @@ -150,6 +184,102 @@ it("discovers stripped Hopper class labels through MCP with literal filtering", }); }); +it("reports exact selected-image class-import identity through MCP", async () => { + const hopper = { id: "hopper", name: "Hopper", version: "1" }; + const native = { id: "native", name: "Native", version: "1" }; + const subject = { + sha256: "a".repeat(64), + path: "/fixture", + format: "mach-o" as const, + architecture: "arm64" as const, + }; + const targetSelection = { + kind: "macho-selected-image-v1" as const, + container: "thin" as const, + offset: 0, + byte_length: 4096, + cpu_type: 0x0100000c, + cpu_subtype: 0x80000002, + container_table_sha256: null, + }; + const profile = createAnalysisProfile(hopper, { + target_selection: targetSelection, + }); + const analysis: AnalysisOperationPort = { + execute: async (operation: AnalysisOperation) => { + if (operation === "list_names") + return resultOk( + createAnalysisExecution( + [ + { address: "0x1000", value: "OBJC_CLASS_$_Document$GOT" }, + { address: "0x2000", value: "OBJC_CLASS_$_Document" }, + ], + hopper, + { subject, analysisProfile: profile }, + ), + ); + if (operation === "inspect_native_dispatch_metadata") + return resultOk( + createAnalysisExecution( + { + target_sha256: subject.sha256, + target_architecture: subject.architecture, + provider: native, + analysis_profile_digest: null, + result: { + db_save_result: null, + target_selection: targetSelection, + objc_class_import_bindings: [ + { + requested_address: "0x1000", + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + library: + "/System/Library/Frameworks/AppKit.framework/AppKit", + weak: false, + addend: "0", + }, + ], + objc_class_import_binding_coverage: { + facet: "objc_class_import_bindings", + status: "complete", + reason: null, + examined: 2, + decoded: 1, + }, + }, + }, + native, + { subject }, + ), + ); + throw new Error(`Unexpected operation: ${operation}`); + }, + }; + const client = await connect(analysis); + const result = jsonResult( + await client.callTool({ name: "get_objc_classes", arguments: {} }), + ); + expect(result).toMatchObject({ + count: 2, + classes: [ + { + address: "0x1000", + name: "OBJC_CLASS_$_Document$GOT", + binding_identity: { + symbol: "_OBJC_CLASS_$_Document", + binding_source: "chained", + library_ordinal: 1, + slot_addresses: ["0x1000"], + }, + }, + { address: "0x2000", name: "OBJC_CLASS_$_Document" }, + ], + identity_coverage: { status: "complete", reason: null }, + }); +}); + const completeOverviewPort: AnalysisOperationPort = { execute: (name) => { switch (name) { diff --git a/tests/boundary/providers/native/objcClassMetadataFallback.test.ts b/tests/boundary/providers/native/objcClassMetadataFallback.test.ts new file mode 100644 index 0000000000..afa8c31ac3 --- /dev/null +++ b/tests/boundary/providers/native/objcClassMetadataFallback.test.ts @@ -0,0 +1,179 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import type { + AnalysisOperation, + AnalysisOperationPort, +} from "../../../../src/application/AnalysisProvider.js"; +import { createAnalysisExecution } from "../../../../src/application/AnalysisProvider.js"; +import { EnhancedTools } from "../../../../src/application/EnhancedTools.js"; +import { + UnsupportedMachoMetadataFormatError, + parseMachoLayout, +} from "../../../../src/native/AppleMachoSelection.js"; +import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; +import { NativeFixtureRunner } from "../../../fixtures/nativeCommands.js"; +import { ok } from "../../../../src/domain/result.js"; + +const thin32MachOHeader = (cpuType: number): Buffer => { + const bytes = Buffer.alloc(84); + bytes.writeUInt32LE(0xfeedface, 0); + bytes.writeUInt32LE(cpuType, 4); + bytes.writeUInt32LE(cpuType === 7 ? 3 : 0, 8); + bytes.writeUInt32LE(2, 12); + bytes.writeUInt32LE(1, 16); + bytes.writeUInt32LE(56, 20); + bytes.writeUInt32LE(1, 28); // LC_SEGMENT + bytes.writeUInt32LE(56, 32); + bytes.write("__TEXT", 36); + bytes.writeUInt32LE(0x1000, 52); // vmaddr + bytes.writeUInt32LE(0x1000, 56); // vmsize + bytes.writeUInt32LE(0, 60); // fileoff + bytes.writeUInt32LE(bytes.length, 64); // filesize + bytes.writeUInt32LE(7, 68); // maxprot + bytes.writeUInt32LE(5, 72); // initprot + return bytes; +}; + +const inspect = async ( + bytes: Buffer, + architecture: "x86" | "arm" | "x86_64", +) => { + const directory = await mkdtemp(join(tmpdir(), "rea-objc-macho-fallback-")); + const path = join(directory, "fixture"); + await writeFile(path, bytes); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const target = { + path, + sha256, + kind: "executable" as const, + format: "mach-o" as const, + architecture, + availableArchitectures: [architecture], + }; + const nativeClient = new NativeMacOSProvider( + {}, + new NativeFixtureRunner(), + "darwin", + ).createClient(target); + try { + const hopper = { id: "hopper", name: "Hopper", version: "1" }; + const analysis: AnalysisOperationPort = { + execute: async (operation: AnalysisOperation, parameters, options) => { + if (operation === "list_names") + return ok( + createAnalysisExecution( + [{ address: "0x1000", value: "OBJC_CLASS_$_Known" }], + hopper, + { subject: target }, + ), + ); + return nativeClient.execute(operation, parameters, options); + }, + }; + return await new EnhancedTools(analysis).execute("get_objc_classes", {}); + } finally { + let closeResult: Awaited> | undefined; + let closeError: unknown; + try { + closeResult = await nativeClient.close(); + } catch (cause: unknown) { + closeError = cause; + } + await rm(directory, { recursive: true, force: true }); + if (closeError !== undefined) throw closeError; + expect(closeResult).toMatchObject({ ok: true }); + } +}; + +describe("native Objective-C class metadata capability fallback", () => { + it.each([ + { architecture: "x86" as const, cpuType: 7 }, + { architecture: "arm" as const, cpuType: 12 }, + ])( + "keeps $architecture names when a valid 32-bit executable is unsupported", + async ({ architecture, cpuType }) => { + const result = await inspect(thin32MachOHeader(cpuType), architecture); + expect(result).toMatchObject({ + ok: true, + value: { + count: 1, + classes: [{ address: "0x1000", name: "OBJC_CLASS_$_Known" }], + identity_coverage: { status: "unknown" }, + }, + }); + }, + ); + + it("classifies exactly the complete 28-byte 32-bit header as unsupported", () => { + const bytes = Buffer.alloc(28); + bytes.writeUInt32LE(0xfeedface, 0); + bytes.writeUInt32LE(7, 4); + bytes.writeUInt32LE(3, 8); + bytes.writeUInt32LE(2, 12); + let failure: unknown; + try { + parseMachoLayout(bytes, "x86"); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(UnsupportedMachoMetadataFormatError); + }); + + it("keeps truncated 32-bit and 64-bit headers as errors", async () => { + const truncated32 = Buffer.alloc(27); + truncated32.writeUInt32LE(0xfeedface, 0); + const truncated32Result = await inspect(truncated32, "x86"); + expect(truncated32Result).toMatchObject({ + ok: false, + error: { _tag: "ProviderAdapterError" }, + }); + + const truncated64 = Buffer.alloc(31); + truncated64.writeUInt32LE(0xfeedfacf, 0); + truncated64.writeUInt32LE(0x01000007, 4); + const truncated64Result = await inspect(truncated64, "x86_64"); + expect(truncated64Result).toMatchObject({ + ok: false, + error: { _tag: "ProviderAdapterError" }, + }); + }); + + it("does not treat an unknown header as a supported-format capability limit", async () => { + const result = await inspect(Buffer.alloc(32), "x86_64"); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "ProviderAdapterError" }, + }); + }); + + it("classifies a complete big-endian PowerPC64 Mach-O header as unsupported", () => { + const bytes = Buffer.alloc(32); + bytes.writeUInt32BE(0xfeedfacf, 0); + bytes.writeUInt32BE(0x01000012, 4); // CPU_TYPE_POWERPC64 + bytes.writeUInt32BE(0, 8); + bytes.writeUInt32BE(2, 12); + let failure: unknown; + try { + parseMachoLayout(bytes, "powerpc"); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(UnsupportedMachoMetadataFormatError); + + bytes.writeUInt32BE(0x01000007, 4); + let mismatchedCpuFailure: unknown; + try { + parseMachoLayout(bytes, "powerpc"); + } catch (cause: unknown) { + mismatchedCpuFailure = cause; + } + expect(mismatchedCpuFailure).toBeInstanceOf(TypeError); + expect(mismatchedCpuFailure).not.toBeInstanceOf( + UnsupportedMachoMetadataFormatError, + ); + }); +}); diff --git a/tests/fixtures/nativeObjcClassImportMachO.ts b/tests/fixtures/nativeObjcClassImportMachO.ts new file mode 100644 index 0000000000..d39ebd80bd --- /dev/null +++ b/tests/fixtures/nativeObjcClassImportMachO.ts @@ -0,0 +1,69 @@ +/** Source-authored thin arm64 Mach-O with one classic dyld class-symbol bind. */ +export const nativeObjcClassImportMachO = () => { + const address = "0x100001000"; + const unboundAddress = "0x100001008"; + const bindStream = Buffer.from([ + 0x11, // Set dylib ordinal 1. + 0x40, + ...Buffer.from("_OBJC_CLASS_$_Document\0"), + 0x51, // Set pointer bind type. + 0x71, + 0x00, // Segment 1, offset 0. + 0x90, // Bind once. + 0x00, + ]); + const bindOffset = 0x300; + const slotOffset = 0x400; + const bytes = Buffer.alloc(slotOffset + 16); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + view.setUint32(0, 0xfeedfacf, true); + view.setUint32(4, 0x0100000c, true); // CPU_TYPE_ARM64 + view.setUint32(8, 0, true); // CPU_SUBTYPE_ARM64_ALL + view.setUint32(12, 2, true); // MH_EXECUTE + view.setUint32(16, 4, true); + view.setUint32(20, 72 + 72 + 48 + 48, true); + + const writeSegment = ( + at: number, + name: string, + vmAddress: bigint, + fileOffset: number, + fileSize: number, + protection: number, + ) => { + view.setUint32(at, 0x19, true); // LC_SEGMENT_64 + view.setUint32(at + 4, 72, true); + bytes.write(name, at + 8, "ascii"); + view.setBigUint64(at + 24, vmAddress, true); + view.setBigUint64(at + 32, BigInt(fileSize), true); + view.setBigUint64(at + 40, BigInt(fileOffset), true); + view.setBigUint64(at + 48, BigInt(fileSize), true); + view.setUint32(at + 56, 7, true); + view.setUint32(at + 60, protection, true); + }; + + const text = 32; + writeSegment(text, "__TEXT", 0x100000000n, 0, slotOffset, 5); + const data = text + 72; + writeSegment(data, "__DATA", 0x100001000n, slotOffset, 16, 3); + + const dylib = data + 72; + const library = Buffer.from( + "/System/Library/Frameworks/AppKit.framework/AppKit\0", + ); + const dylibSize = Math.ceil((24 + library.length) / 8) * 8; + view.setUint32(20, 72 + 72 + dylibSize + 48, true); + view.setUint32(dylib, 0x0c, true); // LC_LOAD_DYLIB + view.setUint32(dylib + 4, dylibSize, true); + view.setUint32(dylib + 8, 24, true); + bytes.set(library, dylib + 24); + + const dyld = dylib + dylibSize; + view.setUint32(dyld, 0x80000022, true); // LC_DYLD_INFO_ONLY + view.setUint32(dyld + 4, 48, true); + view.setUint32(dyld + 16, bindOffset, true); + view.setUint32(dyld + 20, bindStream.length, true); + bytes.set(bindStream, bindOffset); + + return { bytes, address, unboundAddress }; +}; From 949966712e68d8857a50a165ecddf04c2d7e6d3b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:14:03 +0800 Subject: [PATCH 23/35] fix(files): preserve physical read failures across late cancellation Tag physical read outcomes at the owning boundary and preserve opaque caller cancellation reasons through JSON pipelines and reference imports. Share named bounded-read options across consumers and retain failed cleanup ownership for retry. Closes #5686 --- src/application/EvidenceBundleFiles.ts | 37 +- src/application/JsonFiles.ts | 47 ++- src/application/JsonInputFile.ts | 330 ++++++++++++------ src/application/LinuxHopperPublisher.ts | 4 +- .../ReferenceSourceImportPolicy.ts | 58 ++- src/application/RegularFileRead.ts | 85 +++-- src/application/SetupSkill.ts | 9 +- .../analysisView/evidenceFileSource.test.ts | 144 +++++++- .../analysisView/evidenceFileSource.ts | 39 ++- src/application/process/ProcessCli.ts | 21 +- src/cliJsonInput.ts | 46 ++- src/dotnet/ManagedSnapshot.ts | 2 +- src/ghidra/GhidraClient.ts | 16 +- src/ghidra/GhidraDosSnapshot.ts | 10 +- src/native/AppleDispatchMetadata.ts | 10 +- src/process/BoundedFileBytes.ts | 35 +- .../filesystem/boundedFileBytes.test.ts | 15 +- .../cliJsonInputCancellation.test.ts | 62 +++- .../filesystem/evidenceBundleFiles.test.ts | 111 +++++- .../filesystem/processCliFileInput.test.ts | 75 ++++ .../filesystem/referenceSourceImport.test.ts | 264 +++++++++++++- tests/support/failFileReadAfterAbort.ts | 40 +++ 22 files changed, 1203 insertions(+), 257 deletions(-) create mode 100644 tests/support/failFileReadAfterAbort.ts diff --git a/src/application/EvidenceBundleFiles.ts b/src/application/EvidenceBundleFiles.ts index dd1715c96f..df421ecb97 100644 --- a/src/application/EvidenceBundleFiles.ts +++ b/src/application/EvidenceBundleFiles.ts @@ -22,7 +22,6 @@ import { canonicalJsonParts, } from "../domain/jsonSerialization.js"; import { readJsonFile, writeTextParts } from "./JsonFiles.js"; -import { RegularFileCleanupFailure } from "./RegularFileRead.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; type EvidenceReadFailure = @@ -58,23 +57,27 @@ const readBundle = async ( resources?: ArtifactResourceScope, ): Promise> => { if (signal?.aborted) - return err(new AnalysisCancelledError("import_evidence_bundle")); - const loaded = await readJsonFile(path, signal, resources).catch( - (cause: unknown) => { - // A failed close transfers a live handle owner; preserve that failure. - if ( - signal?.aborted === true && - !(cause instanceof RegularFileCleanupFailure) - ) - return err( - new AnalysisCancelledError("import_evidence_bundle", { cause }), - ); - throw cause; - }, - ); - if (!loaded.ok) return loaded; + return err( + new AnalysisCancelledError("import_evidence_bundle", { + cause: signal.reason, + }), + ); + const loaded = await readJsonFile(path, signal, resources); + if (!loaded.ok) { + if (loaded.error instanceof AnalysisCancelledError) + return err( + new AnalysisCancelledError("import_evidence_bundle", { + cause: loaded.error.cause, + }), + ); + return loaded; + } if (signal?.aborted) - return err(new AnalysisCancelledError("import_evidence_bundle")); + return err( + new AnalysisCancelledError("import_evidence_bundle", { + cause: signal.reason, + }), + ); let bundle: EvidenceBundle; try { bundle = parseBundle(loaded.value); diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts index 66ae8487f0..2e81dd8fcf 100644 --- a/src/application/JsonFiles.ts +++ b/src/application/JsonFiles.ts @@ -19,6 +19,8 @@ import { } from "../domain/analysisErrorCore.js"; import { err, ok, type Result } from "../domain/result.js"; import { + JsonInputPhysicalReadFailure, + JsonInputCancellationFailure, readJsonInputFile, readJsonObjectFile, type JsonObjectMemberReader, @@ -46,7 +48,10 @@ export const readJsonFile = async ( signal?: AbortSignal, resources?: ArtifactResourceScope, ): Promise< - Result + Result< + unknown, + EvidenceFileError | AnalysisResourceConstraintError | AnalysisCancelledError + > > => readJsonFileInput( path, @@ -93,7 +98,10 @@ const readJsonFileInput = async ( >, signal?: AbortSignal, ): Promise< - Result + Result< + Value, + EvidenceFileError | AnalysisResourceConstraintError | AnalysisCancelledError + > > => { const requestedPath = absoluteCallerPath(path); try { @@ -116,7 +124,25 @@ const readJsonFileInput = async ( } catch (cause: unknown) { if (cause instanceof RegularFileCleanupFailure) throw cause; if (cause instanceof AnalysisResourceConstraintError) return err(cause); - if (signal?.aborted === true) throw cause; + if (cause instanceof JsonInputCancellationFailure) + return err( + new AnalysisCancelledError("read_evidence_file", { + cause: cause.reason, + }), + ); + if (cause instanceof JsonInputPhysicalReadFailure) + return err( + new EvidenceFileError("read", "io", { + cause: cause.physicalCause, + path: requestedPath, + }), + ); + if (isJsonReadCancellation(cause, signal)) + return err( + new AnalysisCancelledError("read_evidence_file", { + cause: signal?.reason, + }), + ); return err( new EvidenceFileError( "read", @@ -132,6 +158,21 @@ const readJsonFileInput = async ( } }; +const isJsonReadCancellation = ( + cause: unknown, + signal?: AbortSignal, +): boolean => { + if (signal?.aborted !== true) return false; + if (cause === signal.reason) return true; + return ( + cause instanceof Error && + cause.name === "AbortError" && + "code" in cause && + cause.code === "ABORT_ERR" && + cause.cause === signal.reason + ); +}; + /** Atomically write text to the caller-supplied path. */ export const writeTextFile = async ( encoded: string, diff --git a/src/application/JsonInputFile.ts b/src/application/JsonInputFile.ts index 2e93d0a679..fe5e0feace 100644 --- a/src/application/JsonInputFile.ts +++ b/src/application/JsonInputFile.ts @@ -1,5 +1,6 @@ import { constants } from "node:buffer"; import { createHash } from "node:crypto"; +import type { Stats } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import { pipeline } from "node:stream/promises"; import { getHeapStatistics } from "node:v8"; @@ -10,10 +11,17 @@ import parser from "stream-json/parser.js"; import streamValues from "stream-json/streamers/stream-values.js"; import streamArray from "stream-json/streamers/stream-array.js"; -import { RegularFileChangedError } from "../filesystem/RegularFile.js"; +import { + assertRegularFileActive, + RegularFileChangedError, +} from "../filesystem/RegularFile.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { sealOwnedJsonInput } from "./OwnedJsonInput.js"; -import { withRegularFile } from "./RegularFileRead.js"; +import { + RegularFileReadFailure, + withRegularFile, + type RegularFileReadOptions, +} from "./RegularFileRead.js"; import { JSON_BYTE_ORDER_MARK_MESSAGE, JSON_INPUT_RESOURCE_REMEDIATION, @@ -34,6 +42,74 @@ const PACKED_TOKEN_PART_CODE_UNITS = 64 * 1024; // This selects an implementation; larger files remain accepted through streaming. const NATIVE_PARSE_READ_BUDGET = 8 * 1024 * 1024; +/** Physical descriptor failure remains primary if cancellation arrives afterward. */ +export class JsonInputPhysicalReadFailure extends Error { + readonly _tag = "JsonInputPhysicalReadFailure"; + readonly physicalCause: unknown; + constructor(physicalCause: unknown) { + super("JSON input descriptor read failed"); + this.physicalCause = physicalCause; + } +} + +/** Explicit cancellation fact; the caller's value is opaque and retained as data. */ +export class JsonInputCancellationFailure extends Error { + readonly _tag = "JsonInputCancellationFailure"; + constructor(readonly reason: unknown) { + super( + reason instanceof Error ? reason.message : "JSON input read cancelled", + ); + } +} + +interface JsonInputPhysicalReadObservation { + failure?: JsonInputPhysicalReadFailure; +} + +const readPhysicalJsonInput = async ( + observation: JsonInputPhysicalReadObservation, + read: () => Promise, +): Promise => { + try { + return await read(); + } catch (physicalCause: unknown) { + const failure = new JsonInputPhysicalReadFailure(physicalCause); + observation.failure = failure; + throw failure; + } +}; + +const withJsonInputFile = ( + path: string, + read: (handle: FileHandle, stats: Stats) => Promise, + options: Pick, +): Promise => + withRegularFile(path, read, { + ...options, + cancellationFailure: (reason) => new JsonInputCancellationFailure(reason), + }).catch((cause: unknown) => { + if (cause instanceof RegularFileReadFailure) throw cause.operationCause; + throw cause; + }); + +const withJsonInputPipeline = async ( + signal: AbortSignal | undefined, + run: (signal?: AbortSignal) => Promise, +): Promise => { + if (signal === undefined) return run(); + const controller = new AbortController(); + const cancel = (): void => { + controller.abort(new JsonInputCancellationFailure(signal.reason)); + }; + if (signal.aborted) cancel(); + else signal.addEventListener("abort", cancel, { once: true }); + try { + return await run(controller.signal); + } finally { + signal.removeEventListener("abort", cancel); + } +}; + /** Heap a whole-file native JSON.parse may need: the assembler's 64x container expansion plus byte and text copies. */ export const nativeJsonParseRequiredBytes = ( inputBytes: number, @@ -66,10 +142,11 @@ export const readJsonObjectFile = ( signal?: AbortSignal, resources?: ArtifactResourceScope, ): Promise> => - withRegularFile( + withJsonInputFile( path, async (handle, stats) => { let previousDigest: string | undefined; + const physicalRead: JsonInputPhysicalReadObservation = {}; try { const native = await readNativeJson( handle, @@ -77,6 +154,8 @@ export const readJsonObjectFile = ( path, operation, signal, + false, + physicalRead, ); if (native !== undefined) { if (!native.ok) return native; @@ -92,7 +171,7 @@ export const readJsonObjectFile = ( for (const members of scans) for (const [key, member] of members) { if (!Object.hasOwn(value, key)) continue; - signal?.throwIfAborted(); + assertRegularFileActive(signal); member.begin(); const selected: unknown = Reflect.get(value, key); if (member.mode === "array" && !Array.isArray(selected)) { @@ -105,7 +184,7 @@ export const readJsonObjectFile = ( Array.isArray(selected) ? selected : [selected]) { - signal?.throwIfAborted(); + assertRegularFileActive(signal); try { await member.accept(item); } catch (cause: unknown) { @@ -123,68 +202,80 @@ export const readJsonObjectFile = ( let consume: | ((token: parser.Token) => typeof none | Promise) | undefined; - await pipeline( - decodedChunks(handle, stats.size, path, signal, (bytes) => { - hash.update(bytes); - }), - asStream( - fun( - parser({ - jsonStreaming: false, - streamValues: false, - packKeys: false, - packStrings: false, - packNumbers: false, - }), - packJsonScalars(operation, stats.size, path), - (token: parser.Token): typeof none | Promise => { - const opens = - token.name === "startObject" || token.name === "startArray"; - const closes = - token.name === "endObject" || token.name === "endArray"; - if (depth === 0) { - rootObject = token.name === "startObject"; - } else if ( - rootObject && - depth === 1 && - token.name === "keyValue" - ) { - key = token.value; - } else if (rootObject && depth === 1 && !closes) { - const member = - key === undefined ? undefined : members.get(key); - consume = undefined; - if (member !== undefined) { - member.begin(); - if ( - member.mode === "array" && - token.name !== "startArray" - ) { - member.reject( - new TypeError(`JSON member ${key} must be an array`), - ); - } else { - consume = memberConsumer( - member, - operation, - stats.size, - path, - ); - } - } - key = undefined; - } - const selected = consume; - if (opens) depth += 1; - if (closes) depth -= 1; - if (rootObject && depth === 1 && token.name !== "keyValue") - consume = undefined; - if (selected === undefined) return none; - return selected(token); + await withJsonInputPipeline(signal, (pipelineSignal) => + pipeline( + decodedChunks( + handle, + stats.size, + path, + signal, + (bytes) => { + hash.update(bytes); }, + physicalRead, ), + asStream( + fun( + parser({ + jsonStreaming: false, + streamValues: false, + packKeys: false, + packStrings: false, + packNumbers: false, + }), + packJsonScalars(operation, stats.size, path), + (token: parser.Token): typeof none | Promise => { + const opens = + token.name === "startObject" || + token.name === "startArray"; + const closes = + token.name === "endObject" || token.name === "endArray"; + if (depth === 0) { + rootObject = token.name === "startObject"; + } else if ( + rootObject && + depth === 1 && + token.name === "keyValue" + ) { + key = token.value; + } else if (rootObject && depth === 1 && !closes) { + const member = + key === undefined ? undefined : members.get(key); + consume = undefined; + if (member !== undefined) { + member.begin(); + if ( + member.mode === "array" && + token.name !== "startArray" + ) { + member.reject( + new TypeError( + `JSON member ${key} must be an array`, + ), + ); + } else { + consume = memberConsumer( + member, + operation, + stats.size, + path, + ); + } + } + key = undefined; + } + const selected = consume; + if (opens) depth += 1; + if (closes) depth -= 1; + if (rootObject && depth === 1 && token.name !== "keyValue") + consume = undefined; + if (selected === undefined) return none; + return selected(token); + }, + ), + ), + pipelineSignal === undefined ? {} : { signal: pipelineSignal }, ), - signal === undefined ? {} : { signal }, ); if (!rootObject) return err( @@ -197,6 +288,7 @@ export const readJsonObjectFile = ( } return ok(undefined); } catch (cause: unknown) { + if (physicalRead.failure !== undefined) throw physicalRead.failure; const failure = jsonReadFailure(cause, operation, stats.size, path); if (failure !== undefined) return err(failure); throw cause; @@ -256,11 +348,12 @@ export const readJsonInputFile = ( resources?: ArtifactResourceScope, snapshot = false, ): Promise> => - withRegularFile( + withJsonInputFile( path, async (handle, stats) => { let found = false; let value: unknown; + const physicalRead: JsonInputPhysicalReadObservation = {}; try { const native = await readNativeJson( handle, @@ -269,44 +362,59 @@ export const readJsonInputFile = ( operation, signal, snapshot, + physicalRead, ); if (native !== undefined) return native.ok && snapshot ? await sealOwnedJsonInput(native.value, operation, signal) : native; - await pipeline( - decodedChunks(handle, stats.size, path, signal), - // Fuse synchronous token stages so only file chunks cross the async - // stream boundary. Admission still precedes scalar/value assembly. - asStream( - fun( - parser({ - jsonStreaming: false, - streamValues: false, - packKeys: false, - packStrings: false, - packNumbers: false, - }), - packJsonScalars(operation, stats.size, path), - admitJsonAssembly(operation, stats.size, path, snapshot ? 2 : 1), - streamValues(), - (row: unknown): typeof none => { - if ( - found || - typeof row !== "object" || - row === null || - !("key" in row) || - row.key !== 0 || - !("value" in row) - ) - throw new Error("Unexpected JSON parser result"); - found = true; - value = row.value; - return none; - }, + await withJsonInputPipeline(signal, (pipelineSignal) => + pipeline( + decodedChunks( + handle, + stats.size, + path, + signal, + undefined, + physicalRead, ), + // Fuse synchronous token stages so only file chunks cross the async + // stream boundary. Admission still precedes scalar/value assembly. + asStream( + fun( + parser({ + jsonStreaming: false, + streamValues: false, + packKeys: false, + packStrings: false, + packNumbers: false, + }), + packJsonScalars(operation, stats.size, path), + admitJsonAssembly( + operation, + stats.size, + path, + snapshot ? 2 : 1, + ), + streamValues(), + (row: unknown): typeof none => { + if ( + found || + typeof row !== "object" || + row === null || + !("key" in row) || + row.key !== 0 || + !("value" in row) + ) + throw new Error("Unexpected JSON parser result"); + found = true; + value = row.value; + return none; + }, + ), + ), + pipelineSignal === undefined ? {} : { signal: pipelineSignal }, ), - signal === undefined ? {} : { signal }, ); return found ? snapshot @@ -314,6 +422,7 @@ export const readJsonInputFile = ( : ok(value) : err(invalidJson(operation, "JSON document is empty")); } catch (cause: unknown) { + if (physicalRead.failure !== undefined) throw physicalRead.failure; const failure = jsonReadFailure(cause, operation, stats.size, path); if (failure !== undefined) return err(failure); throw cause; @@ -526,6 +635,7 @@ async function* decodedChunks( path: string, signal: AbortSignal | undefined, observe?: (bytes: Uint8Array) => void, + physicalRead: JsonInputPhysicalReadObservation = {}, ): AsyncGenerator { const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); let leading = true; @@ -539,14 +649,16 @@ async function* decodedChunks( const bytes = Buffer.allocUnsafe(READ_CHUNK_BYTES); let observedBytes = 0; while (true) { - signal?.throwIfAborted(); - const read = await handle.read( - bytes, - 0, - Math.min(bytes.length, admittedBytes - observedBytes + 1), - observedBytes, + assertRegularFileActive(signal); + const read = await readPhysicalJsonInput(physicalRead, () => + handle.read( + bytes, + 0, + Math.min(bytes.length, admittedBytes - observedBytes + 1), + observedBytes, + ), ); - signal?.throwIfAborted(); + assertRegularFileActive(signal); if (read.bytesRead === 0) break; observedBytes += read.bytesRead; if (observedBytes > admittedBytes) throw new RegularFileChangedError(path); @@ -569,13 +681,16 @@ const readPrefix = async ( handle: FileHandle, size: number, signal: AbortSignal | undefined, + physicalRead: JsonInputPhysicalReadObservation, ): Promise<{ readonly bytes: Buffer; readonly complete: boolean }> => { const bytes = Buffer.allocUnsafe(size); let offset = 0; while (offset < bytes.length) { - signal?.throwIfAborted(); - const read = await handle.read(bytes, offset, bytes.length - offset, null); - signal?.throwIfAborted(); + assertRegularFileActive(signal); + const read = await readPhysicalJsonInput(physicalRead, () => + handle.read(bytes, offset, bytes.length - offset, null), + ); + assertRegularFileActive(signal); if (read.bytesRead === 0) return { bytes: bytes.subarray(0, offset), complete: true }; offset += read.bytesRead; @@ -591,9 +706,10 @@ const readNativeJson = async ( operation: string, signal?: AbortSignal, snapshot = false, + physicalRead: JsonInputPhysicalReadObservation = {}, ): Promise | undefined> => { if (!nativeParseFits(size, snapshot)) return undefined; - const prefix = await readPrefix(handle, size + 1, signal); + const prefix = await readPrefix(handle, size + 1, signal, physicalRead); if (!prefix.complete || prefix.bytes.length !== size) throw new RegularFileChangedError(path); const parsed = parseUtf8Json( diff --git a/src/application/LinuxHopperPublisher.ts b/src/application/LinuxHopperPublisher.ts index eb63d578e1..9d6f1451fb 100644 --- a/src/application/LinuxHopperPublisher.ts +++ b/src/application/LinuxHopperPublisher.ts @@ -926,7 +926,9 @@ export class LinuxHopperPublisher { "integrity", "Retained Hopper evidence exceeds its resource byte budget.", ); - const bytes = await readBoundedFileBytes(handle, stats.size, signal); + const bytes = await readBoundedFileBytes(handle, stats.size, { + signal, + }); if (bytes === undefined) throw new HopperReleaseError( "integrity", diff --git a/src/application/ReferenceSourceImportPolicy.ts b/src/application/ReferenceSourceImportPolicy.ts index ca314bbd80..fabf360700 100644 --- a/src/application/ReferenceSourceImportPolicy.ts +++ b/src/application/ReferenceSourceImportPolicy.ts @@ -5,6 +5,7 @@ import ignore from "ignore"; import { readRegularFileText, RegularFileCleanupFailure, + RegularFileReadFailure, retryRegularFileCleanup, } from "./RegularFileRead.js"; @@ -17,6 +18,7 @@ import { type ReferenceSourceImportOptions, } from "./ReferenceSourceImportTypes.js"; import { rootFilesystemFailure } from "../reference/ReferenceSourceReaderErrors.js"; +import { regularFileCancellation } from "../filesystem/OwnedRegularFile.js"; /** Validated inputs ready for filesystem traversal. */ export interface PreparedReferenceSourceImport { @@ -34,6 +36,25 @@ const failure = ( message, }); +class IgnorePolicyReadCancelled extends Error { + constructor(readonly reason: unknown) { + super("Reference source ignore policy read was cancelled", { + cause: reason, + }); + this.name = "IgnorePolicyReadCancelled"; + } +} + +const isIgnorePolicyCancellation = (cause: unknown): boolean => + cause instanceof IgnorePolicyReadCancelled; + +const ignorePolicyCause = (cause: unknown): unknown => + cause instanceof IgnorePolicyReadCancelled + ? cause.reason + : cause instanceof RegularFileReadFailure + ? cause.operationCause + : cause; + const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true; const addMarkedPatterns = ( @@ -102,30 +123,46 @@ const buildIgnored = async ( throw cause; present = false; } - signal?.throwIfAborted(); + if (signal?.aborted === true) + throw new IgnorePolicyReadCancelled(signal.reason); if (present) { let text: string; try { - text = await readRegularFileText(policyPath, { signal }); + text = await readRegularFileText(policyPath, { + signal, + cancellationFailure: (reason) => new IgnorePolicyReadCancelled(reason), + }); } catch (cause: unknown) { - if (signal?.aborted === true && cause === signal.reason) throw cause; + if (cause instanceof IgnorePolicyReadCancelled) throw cause; if (!(cause instanceof RegularFileCleanupFailure)) throw cause; const retry = await retryRegularFileCleanup(cause, resources, signal); + const operationCancellation = + cause.outcome.kind === "failed" + ? regularFileCancellation(cause) + : undefined; if (retry.cleanup !== undefined) { const primary = retry.outcome.kind === "failed" ? retry.outcome.cause : cause.cleanupCause; + const primaryCause = ignorePolicyCause(primary); return err({ ...failure( - signal?.aborted === true ? "cancelled" : "io", - `Reference source ignore policy could not be read at ${policyPath}: ${errorMessage(primary)}`, + isIgnorePolicyCancellation(primary) || + operationCancellation !== undefined + ? "cancelled" + : "io", + `Reference source ignore policy could not be read at ${policyPath}: ${errorMessage(primaryCause)}`, ), cleanup: retry.cleanup, - cause: primary, + cause: primaryCause, }); } - if (retry.outcome.kind === "failed") throw retry.outcome.cause; + if (retry.outcome.kind === "failed") { + if (operationCancellation !== undefined) + throw new IgnorePolicyReadCancelled(operationCancellation.reason); + throw retry.outcome.cause; + } text = retry.outcome.value; } addMarkedPatterns(ignored, [text], "project-ignored"); @@ -166,12 +203,13 @@ export const prepareReferenceSourceImport = async ( secrets: ignore().add([...options.policy.secretPatterns]), }); } catch (cause: unknown) { + const causeValue = ignorePolicyCause(cause); return err({ ...failure( - options.signal?.aborted === true ? "cancelled" : "io", - `Reference source ignore policy could not be read at ${join(root.value, ".gitignore")}: ${errorMessage(cause)}`, + isIgnorePolicyCancellation(cause) ? "cancelled" : "io", + `Reference source ignore policy could not be read at ${join(root.value, ".gitignore")}: ${errorMessage(causeValue)}`, ), - cause, + cause: causeValue, }); } }); diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index cdbadedc46..65ed2af261 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -10,6 +10,7 @@ import { import { RegularFileCleanupFailure, RegularFileCancelled, + regularFileCancellation, withOwnedRegularFile, type RegularFileOperationOutcome, } from "../filesystem/OwnedRegularFile.js"; @@ -24,6 +25,18 @@ export interface RegularFileReadOptions { readonly resources?: ArtifactResourceScope; /** Runs on the opened file's own state before any byte is read; throw to refuse the read. */ readonly admit?: (state: Stats) => void; + /** Tags cancellation at the bounded reader's cooperative checkpoints. */ + readonly cancellationFailure?: (reason: unknown) => Error; +} + +/** A regular-file failure returned alongside an enabled cancellation factory. */ +export class RegularFileReadFailure extends Error { + readonly _tag = "RegularFileReadFailure"; + + constructor(readonly operationCause: unknown) { + super("Regular file operation failed", { cause: operationCause }); + this.name = this._tag; + } } export class InvalidUtf8FileError extends TypeError { @@ -153,21 +166,40 @@ export const withRegularFile = async ( // Caller reasons are opaque, including objects shaped like file failures. // They do not transfer ownership of any descriptor to this operation. if (signal?.aborted === true && cause === signal.reason) throw cause; - if (cause instanceof RegularFileOperationCancelled) throw cause.reason; + if (cause instanceof RegularFileReadFailure) throw cause; if ( cause instanceof RegularFileCancelled && - cause.owner.handle === admitted - ) - throw cause.reason; - if ( - !(cause instanceof RegularFileCleanupFailure) || - resources === undefined + cause.owner.handle !== admitted ) throw cause; - const retry = await retryRegularFileCleanup(cause, resources, signal); - if (retry.cleanup !== undefined) throw cause; - if (retry.outcome.kind === "failed") throw retry.outcome.cause; - return retry.outcome.value; + if (cause instanceof RegularFileCleanupFailure) { + if (resources === undefined) throw cause; + const retry = await retryRegularFileCleanup(cause, resources, signal); + if (retry.cleanup !== undefined) throw cause; + if (retry.outcome.kind === "failed") { + if (signal?.aborted === true && retry.outcome.cause === signal.reason) + throw signal.reason; + const cancellation = + regularFileCancellation(retry.outcome.cause) ?? + regularFileCancellation(cause); + if (cancellation !== undefined) { + if (options.cancellationFailure !== undefined) + throw options.cancellationFailure(cancellation.reason); + throw cancellation.reason; + } + throw retry.outcome.cause; + } + return retry.outcome.value; + } + const cancellation = regularFileCancellation(cause); + if (cancellation !== undefined) { + if (options.cancellationFailure !== undefined) + throw options.cancellationFailure(cancellation.reason); + throw cancellation.reason; + } + if (options.cancellationFailure !== undefined) + throw new RegularFileReadFailure(cause); + throw cause; } }; const value = await (resources === undefined @@ -175,6 +207,8 @@ export const withRegularFile = async ( : resources.run(operation)); // A retained close may have recovered after the descriptor owner's outcome // was captured. Include that retry and the scope's finalization in cancellation. + if (signal?.aborted === true && options.cancellationFailure !== undefined) + throw options.cancellationFailure(signal.reason); signal?.throwIfAborted(); return value; }; @@ -225,15 +259,26 @@ const readRegularFileContent = ( withRegularFile( path, async (handle, stats) => { - options.admit?.(stats); - const bytes = await readBoundedFileBytes( - handle, - stats.size, - options.signal, - ); - if (bytes === undefined || bytes.length !== stats.size) - throw new RegularFileChangedError(path); - return decode(bytes, stats); + try { + options.admit?.(stats); + const bytes = await readBoundedFileBytes(handle, stats.size, { + ...(options.signal === undefined ? {} : { signal: options.signal }), + cancellationFailure: (reason) => + new RegularFileOperationCancelled(reason), + operationFailure: (cause) => new RegularFileReadFailure(cause), + }); + if (bytes === undefined || bytes.length !== stats.size) + throw new RegularFileChangedError(path); + return decode(bytes, stats); + } catch (cause: unknown) { + if ( + options.cancellationFailure !== undefined && + !(cause instanceof RegularFileOperationCancelled) && + !(cause instanceof RegularFileReadFailure) + ) + throw new RegularFileReadFailure(cause); + throw cause; + } }, options, ); diff --git a/src/application/SetupSkill.ts b/src/application/SetupSkill.ts index 87adb71ab5..02e22bfc8a 100644 --- a/src/application/SetupSkill.ts +++ b/src/application/SetupSkill.ts @@ -53,12 +53,9 @@ const readSkillFileState = async ( withRegularFile( path, async (handle, stats) => { - const bytes = await readBoundedFileBytes( - handle, - stats.size, - undefined, - stats.size, - ); + const bytes = await readBoundedFileBytes(handle, stats.size, { + expectedSize: stats.size, + }); if (bytes === undefined || bytes.length !== stats.size) throw new RegularFileChangedError(path); return { bytes, mode: stats.mode & 0o7777 }; diff --git a/src/application/analysisView/evidenceFileSource.test.ts b/src/application/analysisView/evidenceFileSource.test.ts index 8bad7eed31..ac593127e0 100644 --- a/src/application/analysisView/evidenceFileSource.test.ts +++ b/src/application/analysisView/evidenceFileSource.test.ts @@ -1,11 +1,15 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { analysisViewCaptureEvidence } from "../../../tests/fixtures/analysisView.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { JsonInputPhysicalReadFailure } from "../JsonInputFile.js"; +import { failFileReadAfterAbort } from "../../../tests/support/failFileReadAfterAbort.js"; +import { holdFirstFileRead } from "../../../tests/support/holdFirstFileRead.js"; import { estimateJsonValueBytes, readEvidenceFileSource, @@ -43,12 +47,23 @@ describe("evidence-file source", () => { it("refuses a cancelled request before reading and reads within the caller's resource scope", async () => { const controller = new AbortController(); - controller.abort(); - expect( - await readEvidenceFileSource("/nonexistent/capture.json", { + const reason = new JsonInputPhysicalReadFailure( + new Error("opaque caller reason with physical-failure shape"), + ); + controller.abort(reason); + const cancelled = await readEvidenceFileSource( + "/nonexistent/capture.json", + { signal: controller.signal, - }), - ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + }, + ); + expect(cancelled).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + if (cancelled.ok || !(cancelled.error instanceof AnalysisCancelledError)) + throw new Error("Expected cancelled analysis-view read"); + expect(cancelled.error.cause).toBe(reason); const path = join( await createTestTempDirectory("rea-evidence-file-source-"), "capture.json", @@ -62,3 +77,120 @@ describe("evidence-file source", () => { expect(read.value).toEqual(JSON.parse(JSON.stringify(parent))); }); }); + +describe("evidence-file read failure provenance", () => { + it("keeps a physical null read cause after the analysis-view request aborts", async () => { + const root = await createTestTempDirectory("rea-view-late-io-"); + const path = join(root, "capture.json"); + await writeFile(path, JSON.stringify(analysisViewCaptureEvidence())); + const controller = new AbortController(); + await failFileReadAfterAbort( + path, + controller, + new Error("later abort"), + null, + ); + try { + const result = await readEvidenceFileSource(path, { + signal: controller.signal, + }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "EvidenceFileError", reason: "io", path }, + }); + if (result.ok) throw new Error("Expected physical JSON read failure"); + expect(result.error.cause).toBeNull(); + } finally { + vi.restoreAllMocks(); + } + }); + + it("preserves identical physical and abort causes for analysis-view reads", async () => { + const root = await createTestTempDirectory("rea-view-identical-abort-"); + const path = join(root, "capture.json"); + await writeFile(path, JSON.stringify(analysisViewCaptureEvidence())); + const controller = new AbortController(); + const physical = Object.assign(new Error("descriptor EIO"), { + code: "EIO", + }); + await failFileReadAfterAbort(path, controller, physical, physical); + try { + const result = await readEvidenceFileSource(path, { + signal: controller.signal, + }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "EvidenceFileError", reason: "io" }, + }); + if (result.ok) throw new Error("Expected physical JSON read failure"); + expect(result.error.cause).toBe(physical); + } finally { + vi.restoreAllMocks(); + } + }); + + it("preserves the caller cancellation reason during an analysis-view read", async () => { + const root = await createTestTempDirectory("rea-view-read-cancel-"); + const path = join(root, "capture.json"); + await writeFile(path, JSON.stringify(analysisViewCaptureEvidence())); + const reader = await holdFirstFileRead(path); + const controller = new AbortController(); + const reason = new Error("stop analysis-view read"); + try { + const pending = readEvidenceFileSource(path, { + signal: controller.signal, + }); + await reader.firstRead; + controller.abort(reason); + reader.release(); + const result = await pending; + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + operation: "inspect_analysis_view", + }, + }); + if (result.ok || !(result.error instanceof AnalysisCancelledError)) + throw new Error("Expected cancelled analysis-view read"); + expect(result.error.cause).toBe(reason); + } finally { + controller.abort(reason); + reader.release(); + vi.restoreAllMocks(); + } + }); + + it("does not treat a caller reason with physical-read shape as a physical read", async () => { + const root = await createTestTempDirectory("rea-view-opaque-cancel-"); + const path = join(root, "capture.json"); + await writeFile(path, JSON.stringify(analysisViewCaptureEvidence())); + const reader = await holdFirstFileRead(path); + const controller = new AbortController(); + const reason = new JsonInputPhysicalReadFailure( + new Error("opaque caller reason with physical-failure shape"), + ); + try { + const pending = readEvidenceFileSource(path, { + signal: controller.signal, + }); + await reader.firstRead; + controller.abort(reason); + reader.release(); + const result = await pending; + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError", cause: reason }, + }); + if (result.ok || !(result.error instanceof AnalysisCancelledError)) + throw new Error( + "Expected caller cancellation, not physical I/O failure", + ); + expect(result.error.cause).toBe(reason); + } finally { + controller.abort(reason); + reader.release(); + vi.restoreAllMocks(); + } + }); +}); diff --git a/src/application/analysisView/evidenceFileSource.ts b/src/application/analysisView/evidenceFileSource.ts index 6e644b57ee..a4755234b6 100644 --- a/src/application/analysisView/evidenceFileSource.ts +++ b/src/application/analysisView/evidenceFileSource.ts @@ -16,7 +16,6 @@ import { } from "../../domain/evidenceErrors.js"; import { projectInputIssues } from "../../domain/inputIssueProjection.js"; import { readJsonFile } from "../JsonFiles.js"; -import { RegularFileCleanupFailure } from "../RegularFileRead.js"; /** The request's cancellation and the scope that owns descriptors whose close failed. */ export interface EvidenceFileReadOptions { @@ -79,25 +78,27 @@ export const readEvidenceFileSource = async ( ): Promise> => { const { signal, resources } = options; if (signal?.aborted) - return err(new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION)); - const loaded = await readJsonFile(path, signal, resources).catch( - (cause: unknown) => { - // A failed close transfers a live handle owner; preserve that failure. - if ( - signal?.aborted === true && - !(cause instanceof RegularFileCleanupFailure) - ) - return err( - new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, { - cause, - }), - ); - throw cause; - }, - ); - if (!loaded.ok) return loaded; + return err( + new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, { + cause: signal.reason, + }), + ); + const loaded = await readJsonFile(path, signal, resources); + if (!loaded.ok) { + if (loaded.error instanceof AnalysisCancelledError) + return err( + new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, { + cause: loaded.error.cause, + }), + ); + return loaded; + } if (signal?.aborted) - return err(new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION)); + return err( + new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, { + cause: signal.reason, + }), + ); const headroom = requireAuthenticationHeadroom(loaded.value, path); if (!headroom.ok) return headroom; try { diff --git a/src/application/process/ProcessCli.ts b/src/application/process/ProcessCli.ts index 6849847a2a..7c3fa62a48 100644 --- a/src/application/process/ProcessCli.ts +++ b/src/application/process/ProcessCli.ts @@ -238,17 +238,16 @@ const readJson = async ( operation: "capture_process_scenario" | "compare_process_captures", signal?: AbortSignal, ): Promise => { - if (signal?.aborted) throw new AnalysisCancelledError(operation); - const loaded = await readJsonFile(path, signal).catch((cause: unknown) => { - if ( - signal?.aborted === true && - !(cause instanceof RegularFileCleanupFailure) - ) - throw new AnalysisCancelledError(operation, { cause }); - throw cause; - }); - if (signal?.aborted) throw new AnalysisCancelledError(operation); - if (loaded.ok) return loaded.value; + if (signal?.aborted) + throw new AnalysisCancelledError(operation, { cause: signal.reason }); + const loaded = await readJsonFile(path, signal); + if (!loaded.ok && loaded.error instanceof AnalysisCancelledError) + throw new AnalysisCancelledError(operation, { cause: loaded.error.cause }); + if (loaded.ok) { + if (signal?.aborted) + throw new AnalysisCancelledError(operation, { cause: signal.reason }); + return loaded.value; + } if (loaded.error._tag !== "EvidenceFileError") throw loaded.error; if (loaded.error.reason === "io") throw loaded.error; const detail = describeValidationFailure(loaded.error.cause); diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.ts index 24c54055e0..c7c0a36d72 100644 --- a/src/cliJsonInput.ts +++ b/src/cliJsonInput.ts @@ -1,7 +1,11 @@ import { absoluteCallerPath } from "./filesystem/callerPath.js"; import { sealOwnedJsonInput } from "./application/OwnedJsonInput.js"; -import { readJsonInputFile } from "./application/JsonInputFile.js"; +import { + JsonInputCancellationFailure, + JsonInputPhysicalReadFailure, + readJsonInputFile, +} from "./application/JsonInputFile.js"; import { NonRegularFileReadError } from "./filesystem/RegularFile.js"; import { AnalysisAccessDeniedError, @@ -22,14 +26,18 @@ export const parseCliJsonInput = async ( | { readonly ok: true; readonly value: unknown } | { readonly ok: false; readonly error: JsonValue } > => { - const cancelled = () => ({ + const cancelled = (cause: unknown) => ({ ok: false as const, error: { error: "Application workflow failed", - ...projectAnalysisError(new AnalysisCancelledError(operation)), + ...projectAnalysisError( + new AnalysisCancelledError(operation, { + cause, + }), + ), }, }); - if (signal?.aborted) return cancelled(); + if (signal?.aborted) return cancelled(signal.reason); try { const inline = safeParseJson(value); if (inline.ok) { @@ -52,7 +60,7 @@ export const parseCliJsonInput = async ( undefined, options?.snapshot ?? false, ); - if (parsed.ok && signal?.aborted) return cancelled(); + if (parsed.ok && signal?.aborted) return cancelled(signal.reason); return parsed.ok ? { ok: true, value: parsed.value } : { @@ -68,12 +76,17 @@ export const parseCliJsonInput = async ( }, }; } catch (cause: unknown) { - if ( - signal?.aborted && - (cause === signal.reason || - (cause instanceof Error && cause.name === "AbortError")) - ) - return cancelled(); + if (cause instanceof JsonInputCancellationFailure) + return cancelled(cause.reason); + if (cause instanceof JsonInputPhysicalReadFailure) { + const physicalCause = cause.physicalCause; + const systemCode = accessDeniedSystemCode(physicalCause); + return systemCode === undefined + ? jsonFileError(value, operation, "read-failed", physicalCause) + : jsonAccessDeniedError(value, operation, systemCode, physicalCause); + } + if (signal?.aborted && isJsonInputCancellation(cause, signal)) + return cancelled(signal.reason); const systemCode = accessDeniedSystemCode(cause); if (systemCode !== undefined) return jsonAccessDeniedError(value, operation, systemCode, cause); @@ -93,6 +106,17 @@ export const parseCliJsonInput = async ( } }; +const isJsonInputCancellation = ( + cause: unknown, + signal: AbortSignal, +): boolean => + cause === signal.reason || + (cause instanceof Error && + cause.name === "AbortError" && + "code" in cause && + cause.code === "ABORT_ERR" && + cause.cause === signal.reason); + const accessDeniedSystemCode = ( cause: unknown, ): "EACCES" | "EPERM" | undefined => { diff --git a/src/dotnet/ManagedSnapshot.ts b/src/dotnet/ManagedSnapshot.ts index 4edfd89f84..7d539171f9 100644 --- a/src/dotnet/ManagedSnapshot.ts +++ b/src/dotnet/ManagedSnapshot.ts @@ -61,7 +61,7 @@ export const readManagedSnapshot = async ( const bytes = await readBoundedFileBytes( handle, MAX_MANAGED_SNAPSHOT_BYTES, - signal, + { signal }, ); if (bytes === undefined) throw managedSnapshotCapacityError(operation); signal?.throwIfAborted(); diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index a18f086ab0..b8255f30b2 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -414,12 +414,16 @@ export class GhidraClient { const bytes = await readBoundedFileBytes( handle, maximumBytes ?? initial.size, - signal, - initial.size, - (reason) => - new ArtifactReaderFailure("cancelled", "Snapshot read cancelled", { - cause: reason, - }), + { + signal, + expectedSize: initial.size, + cancellationFailure: (reason) => + new ArtifactReaderFailure( + "cancelled", + "Snapshot read cancelled", + { cause: reason }, + ), + }, ); if (bytes === undefined || bytes.length !== initial.size) throw new RegularFileChangedError(path); diff --git a/src/ghidra/GhidraDosSnapshot.ts b/src/ghidra/GhidraDosSnapshot.ts index 143d715a70..603cfd6a30 100644 --- a/src/ghidra/GhidraDosSnapshot.ts +++ b/src/ghidra/GhidraDosSnapshot.ts @@ -30,13 +30,11 @@ export const attestGhidraDosSnapshot = async ( signal?: AbortSignal, ) => { const headerBytes = Math.min(fileBytes, 64); - const header = await readBoundedFileBytes( - handle, - headerBytes, + const header = await readBoundedFileBytes(handle, headerBytes, { signal, - headerBytes, - cancelled, - ); + expectedSize: headerBytes, + cancellationFailure: cancelled, + }); if (header === undefined || header.length !== headerBytes) throw new RegularFileChangedError(path); const extent = dosMzImageByteLength(header); diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index a54658f3c0..dfcbb5f924 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -271,15 +271,9 @@ export const inspectAppleDispatchMetadata = async ( const before = await handle.stat(); if (before.size > maximumBytes) throw capacityError(); if (signal?.aborted) throw cancelled(); - const read = await readBoundedFileBytes( - handle, - maximumBytes, + const read = await readBoundedFileBytes(handle, maximumBytes, { signal, - ).catch((cause: unknown) => { - // The bounded reader rejects with the caller's exact abort reason. - if (signal?.aborted === true && cause === signal.reason) - throw cancelled(); - throw cause; + cancellationFailure: () => cancelled(), }); if (read === undefined) throw capacityError(); const sha256 = createHash("sha256").update(read).digest("hex"); diff --git a/src/process/BoundedFileBytes.ts b/src/process/BoundedFileBytes.ts index fec1c5d988..a33d17c052 100644 --- a/src/process/BoundedFileBytes.ts +++ b/src/process/BoundedFileBytes.ts @@ -1,13 +1,32 @@ import type { FileHandle } from "node:fs/promises"; +export interface BoundedFileReadOptions { + readonly signal?: AbortSignal | undefined; + readonly expectedSize?: number | undefined; + readonly cancellationFailure?: ((reason: unknown) => Error) | undefined; + readonly operationFailure?: ((cause: unknown) => Error) | undefined; +} + /** Read through EOF, returning undefined on overflow; leave the handle open. */ export const readBoundedFileBytes = async ( handle: FileHandle, maxBytes: number, - signal?: AbortSignal, - expectedSize?: number, - cancellationFailure?: (reason: unknown) => Error, + options: BoundedFileReadOptions = {}, ): Promise => { + const { signal, expectedSize, cancellationFailure, operationFailure } = + options; + const read = async ( + buffer: Buffer, + offset: number, + length: number, + ): Promise<{ readonly bytesRead: number }> => { + try { + return await handle.read(buffer, offset, length, null); + } catch (cause: unknown) { + if (operationFailure !== undefined) throw operationFailure(cause); + throw cause; + } + }; const assertActive = (): void => { if (signal?.aborted && cancellationFailure !== undefined) throw cancellationFailure(signal.reason); @@ -28,11 +47,10 @@ export const readBoundedFileBytes = async ( let total = 0; for (;;) { assertActive(); - const { bytesRead } = await handle.read( + const { bytesRead } = await read( buffer, 0, Math.min(buffer.length, maxBytes - total + 1), - null, ); if (bytesRead === 0) return Buffer.concat(chunks, total); total += bytesRead; @@ -45,12 +63,7 @@ export const readBoundedFileBytes = async ( let total = 0; for (;;) { assertActive(); - const { bytesRead } = await handle.read( - buffer, - total, - expectedSize - total, - null, - ); + const { bytesRead } = await read(buffer, total, expectedSize - total); if (bytesRead === 0) return buffer.subarray(0, total); total += bytesRead; if (total > maxBytes) return undefined; diff --git a/tests/boundary/filesystem/boundedFileBytes.test.ts b/tests/boundary/filesystem/boundedFileBytes.test.ts index 02f34b841f..faa832de19 100644 --- a/tests/boundary/filesystem/boundedFileBytes.test.ts +++ b/tests/boundary/filesystem/boundedFileBytes.test.ts @@ -35,20 +35,21 @@ describe("bounded file byte reads", () => { const controller = new AbortController(); controller.abort(reason); await expect( - readBoundedFileBytes(handle, 5, controller.signal, expectedSize), + readBoundedFileBytes(handle, 5, { + signal: controller.signal, + expectedSize, + }), ).rejects.toBe(reason); const typed = new Error("typed cancellation", { cause: reason }); await expect( - readBoundedFileBytes( - handle, - 5, - controller.signal, + readBoundedFileBytes(handle, 5, { + signal: controller.signal, expectedSize, - (observed) => { + cancellationFailure: (observed) => { expect(observed).toBe(reason); return typed; }, - ), + }), ).rejects.toBe(typed); } } finally { diff --git a/tests/boundary/filesystem/cliJsonInputCancellation.test.ts b/tests/boundary/filesystem/cliJsonInputCancellation.test.ts index 8ecfa23fd7..8812af8b94 100644 --- a/tests/boundary/filesystem/cliJsonInputCancellation.test.ts +++ b/tests/boundary/filesystem/cliJsonInputCancellation.test.ts @@ -1,9 +1,10 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; import { parseCliJsonInput } from "../../../src/cliJsonInput.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { failFileReadAfterAbort } from "../../support/failFileReadAfterAbort.js"; it("reports an already-cancelled inline JSON read as cancellation", async () => { const controller = new AbortController(); @@ -29,6 +30,65 @@ it("cancels a pending regular-file JSON read through the owning reader", async ( }); }); +it.each([ + { label: "undefined", physical: undefined }, + { label: "null", physical: null }, +])( + "classifies a physical $label read rejection before late cancellation", + async ({ physical }) => { + const root = await createTestTempDirectory("rea-json-input-late-io-"); + const path = join(root, "input.json"); + await writeFile(path, '{"value":1}'); + const controller = new AbortController(); + await failFileReadAfterAbort( + path, + controller, + new Error("later cancellation"), + physical, + ); + try { + expect( + await parseCliJsonInput(path, "test-input", controller.signal), + ).toMatchObject({ + ok: false, + error: { + code: "invalid_request", + input_reason: "read-failed", + details: { operation: "test-input" }, + }, + }); + } finally { + vi.restoreAllMocks(); + } + }, +); + +it("keeps a physical rejection when it is the identical object used to abort", async () => { + const root = await createTestTempDirectory("rea-json-input-identical-abort-"); + const path = join(root, "input.json"); + await writeFile(path, '{"value":1}'); + const controller = new AbortController(); + const physical = Object.assign(new Error("descriptor EIO"), { code: "EIO" }); + await failFileReadAfterAbort(path, controller, physical, physical); + try { + expect( + await parseCliJsonInput(path, "test-input", controller.signal), + ).toMatchObject({ + ok: false, + error: { + code: "invalid_request", + input_reason: "read-failed", + details: { + operation: "test-input", + issues: [{ message: expect.stringContaining("EIO") }], + }, + }, + }); + } finally { + vi.restoreAllMocks(); + } +}); + it("cancels cooperative sealing of an owned inline Evidence input", async () => { const controller = new AbortController(); const input = JSON.stringify( diff --git a/tests/boundary/filesystem/evidenceBundleFiles.test.ts b/tests/boundary/filesystem/evidenceBundleFiles.test.ts index 2d80bcd8e9..d94b949c86 100644 --- a/tests/boundary/filesystem/evidenceBundleFiles.test.ts +++ b/tests/boundary/filesystem/evidenceBundleFiles.test.ts @@ -25,6 +25,9 @@ import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import { writeTextParts } from "../../../src/application/JsonFiles.js"; +import { failFileReadAfterAbort } from "../../support/failFileReadAfterAbort.js"; +import { holdFirstFileRead } from "../../support/holdFirstFileRead.js"; +import { JsonInputPhysicalReadFailure } from "../../../src/application/JsonInputFile.js"; import { createEvidenceBundle, serializeEvidenceBundle, @@ -47,30 +50,128 @@ it.each([ async (_, read) => { const root = await createTestTempDirectory("rea-bundle-pre-cancel-"); const controller = new AbortController(); - controller.abort(new Error("cancel before bundle read")); - expect( - await read(join(root, "missing.json"), controller.signal), - ).toMatchObject({ + const reason = new Error("cancel before bundle read"); + controller.abort(reason); + const result = await read(join(root, "missing.json"), controller.signal); + expect(result).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError", operation: "import_evidence_bundle", }, }); + if (result.ok) throw new Error("Expected cancelled bundle read"); + expect(result.error.cause).toBe(reason); expect(await readdir(root)).toEqual([]); }, ); +it.each([ + ["mutable", readEvidenceBundle], + ["immutable", readImmutableEvidenceBundle], +] as const)( + "preserves the caller cancellation reason during %s read", + async (_, read) => { + const root = await createTestTempDirectory("rea-bundle-read-cancel-"); + const path = join(root, "bundle.json"); + await writeFile(path, JSON.stringify(bundle())); + const reader = await holdFirstFileRead(path); + const controller = new AbortController(); + const reason = new Error("stop bundle import"); + try { + const pending = read(path, controller.signal); + await reader.firstRead; + controller.abort(reason); + reader.release(); + const result = await pending; + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCancelledError", + operation: "import_evidence_bundle", + }, + }); + if (result.ok) throw new Error("Expected cancelled bundle read"); + expect(result.error.cause).toBe(reason); + } finally { + controller.abort(reason); + reader.release(); + vi.restoreAllMocks(); + } + }, +); + +it.each([ + ["mutable", readEvidenceBundle], + ["immutable", readImmutableEvidenceBundle], +] as const)( + "keeps the physical null read cause for %s import after cancellation", + async (_, read) => { + const root = await createTestTempDirectory("rea-bundle-late-io-"); + const path = join(root, "bundle.json"); + await writeFile(path, JSON.stringify(bundle())); + const controller = new AbortController(); + await failFileReadAfterAbort( + path, + controller, + new Error("later abort"), + null, + ); + try { + const result = await read(path, controller.signal); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "EvidenceFileError", reason: "io", path }, + }); + if (result.ok) throw new Error("Expected physical JSON read failure"); + expect(result.error.cause).toBeNull(); + } finally { + vi.restoreAllMocks(); + } + }, +); + +it.each([ + ["mutable", readEvidenceBundle], + ["immutable", readImmutableEvidenceBundle], +] as const)( + "keeps an identical physical and abort reason for %s import", + async (_, read) => { + const root = await createTestTempDirectory("rea-bundle-identical-abort-"); + const path = join(root, "bundle.json"); + await writeFile(path, JSON.stringify(bundle())); + const controller = new AbortController(); + const physical = new JsonInputPhysicalReadFailure( + Object.assign(new Error("physical EIO"), { code: "EIO" }), + ); + await failFileReadAfterAbort(path, controller, physical, physical); + try { + const result = await read(path, controller.signal); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "EvidenceFileError", reason: "io" }, + }); + if (result.ok) throw new Error("Expected physical JSON read failure"); + expect(result.error.cause).toBe(physical); + } finally { + vi.restoreAllMocks(); + } + }, +); + it("preserves the failed-close owner when import is also cancelled", async () => { const root = await createTestTempDirectory("rea-import-cancel-cleanup-"); const path = join(root, "bundle.json"); await writeFile(path, JSON.stringify(bundle())); const controller = new AbortController(); const cleanupCause = new Error("fixture close failure"); + const cancellationReason = new JsonInputPhysicalReadFailure( + new Error("opaque cancellation reason"), + ); const close = vi .spyOn(OwnedFileHandle.prototype, "close") .mockImplementationOnce(async () => { - controller.abort(new Error("cancel during cleanup")); + controller.abort(cancellationReason); throw cleanupCause; }); let failure: unknown; diff --git a/tests/boundary/filesystem/processCliFileInput.test.ts b/tests/boundary/filesystem/processCliFileInput.test.ts index 07295b15c5..578d59d175 100644 --- a/tests/boundary/filesystem/processCliFileInput.test.ts +++ b/tests/boundary/filesystem/processCliFileInput.test.ts @@ -18,6 +18,8 @@ import * as jsonFiles from "../../../src/application/JsonFiles.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { holdFirstFileRead } from "../../support/holdFirstFileRead.js"; +import { failFileReadAfterAbort } from "../../support/failFileReadAfterAbort.js"; +import { JsonInputPhysicalReadFailure } from "../../../src/application/JsonInputFile.js"; describe("process CLI error projection", () => { it.each([ @@ -172,6 +174,79 @@ describe("process CLI file cancellation", () => { ); }); +it.each( + [ + { + label: "EIO", + physical: Object.assign(new Error("physical descriptor failure"), { + code: "EIO", + }), + }, + { label: "null", physical: null }, + { label: "undefined", physical: undefined }, + ].flatMap((failure) => [ + { ...failure, reader: "native" as const }, + { ...failure, reader: "streamed" as const }, + ]), +)( + "preserves a $reader descriptor $label when cancellation arrives after the read fails", + async ({ physical, reader }) => { + const root = await createTestTempDirectory("rea-process-cli-late-io-"); + const path = join(root, "scenario.json"); + const input = JSON.stringify({ + executable: process.execPath, + arguments: [], + }); + await writeFile(path, reader === "streamed" ? stream(input) : input); + const controller = new AbortController(); + await failFileReadAfterAbort( + path, + controller, + new Error("later caller cancellation"), + physical, + ); + try { + expect( + await captureProcessScenarioFile(path, process.env, controller.signal), + ).toMatchObject({ + code: "execution_failure", + category: "execution_failure", + retryable: false, + details: { operation: "read", reason: "io", path }, + }); + } finally { + vi.restoreAllMocks(); + } + }, +); + +it("keeps a physical read primary when its cause is also the abort reason", async () => { + const root = await createTestTempDirectory( + "rea-process-cli-identical-abort-", + ); + const path = join(root, "scenario.json"); + await writeFile( + path, + JSON.stringify({ executable: process.execPath, arguments: [] }), + ); + const controller = new AbortController(); + const physical = new JsonInputPhysicalReadFailure( + Object.assign(new Error("descriptor EIO"), { code: "EIO" }), + ); + await failFileReadAfterAbort(path, controller, physical, physical); + try { + expect( + await captureProcessScenarioFile(path, process.env, controller.signal), + ).toMatchObject({ + code: "execution_failure", + category: "execution_failure", + details: { operation: "read", reason: "io", path }, + }); + } finally { + vi.restoreAllMocks(); + } +}); + describe("process CLI input diagnostics", () => { it("keeps physical file I/O failures in the execution-error projection", async () => { const path = "/selected/scenario.json"; diff --git a/tests/boundary/filesystem/referenceSourceImport.test.ts b/tests/boundary/filesystem/referenceSourceImport.test.ts index 65c6d4b22d..9cb11b5053 100644 --- a/tests/boundary/filesystem/referenceSourceImport.test.ts +++ b/tests/boundary/filesystem/referenceSourceImport.test.ts @@ -1,5 +1,5 @@ import fs from "node:fs"; -import { chmod, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { chmod, mkdir, open, rm, symlink, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { add, commit, init } from "isomorphic-git"; @@ -8,6 +8,11 @@ import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { normalizeHistoricalSourceParseFailures } from "../../../src/application/ReferenceSourceImport.js"; +import { prepareReferenceSourceImport } from "../../../src/application/ReferenceSourceImportPolicy.js"; +import { + readRegularFileText, + RegularFileReadFailure, +} from "../../../src/application/RegularFileRead.js"; import { importReferenceSource } from "../../support/referenceSourceResourceScope.js"; import { parseReferenceSourceEntries, @@ -15,6 +20,9 @@ import { } from "../../../src/application/ReferenceSourceImportEntries.js"; import type { ReferenceSourceRead } from "../../../src/reference/ReferenceSourceReaderTypes.js"; import { createHistoricalSourceManifest } from "../../../src/domain/referenceSourceGraph.js"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; +import { RegularFileCleanupFailure } from "../../../src/filesystem/OwnedRegularFile.js"; const fixture = async (parent: string, name: string): Promise => { const root = join(parent, name); @@ -88,6 +96,260 @@ describe("reference source import projections", () => { }); }); +describe("reference source ignore policy read failures", () => { + it("preserves a late-aborted physical admission failure identical to the reason", async () => { + const root = await createTestTempDirectory("rea-reference-admission-read-"); + const path = join(root, "input.txt"); + await writeFile(path, "source\n"); + const controller = new AbortController(); + const physical = Object.assign(new Error("controlled admission failure"), { + code: "EIO", + }); + const result = readRegularFileText(path, { + signal: controller.signal, + cancellationFailure: (reason) => + new Error("owned cancellation checkpoint", { cause: reason }), + admit: () => { + controller.abort(physical); + throw physical; + }, + }); + + await expect(result).rejects.toMatchObject({ + _tag: new RegularFileReadFailure(physical)._tag, + operationCause: physical, + cause: physical, + }); + }); + + it.each([ + { + name: "distinct physical I/O failure", + physical: "error" as const, + sameAsAbortReason: false, + expected: "io", + }, + { + name: "physical failure identical to the abort reason", + physical: "error" as const, + sameAsAbortReason: true, + expected: "io", + }, + { + name: "non-Error physical failure", + physical: "string" as const, + sameAsAbortReason: false, + expected: "io", + }, + { + name: "cancellation after a successful physical read", + physical: "cancel-after-read" as const, + sameAsAbortReason: false, + expected: "cancelled", + }, + ])( + "preserves $name when the signal is aborted", + async ({ physical, sameAsAbortReason, expected }) => { + const root = await createTestTempDirectory("rea-reference-ignore-read-"); + const path = join(root, ".gitignore"); + await writeFile(path, "ignored/\n"); + const controller = new AbortController(); + const abortReason = new Error("caller cancelled"); + const readFailure: unknown = + physical === "string" + ? "controlled non-Error read failure" + : Object.assign(new Error("controlled read failure"), { + code: "EIO", + }); + const probe = await open(path, "r"); + const identity = await probe.stat(); + const prototype = probe.constructor.prototype as { + read: (...arguments_: never[]) => Promise; + }; + await probe.close(); + const originalRead = prototype.read; + let intercepted = false; + const read = async function ( + this: typeof probe, + ...arguments_: never[] + ): Promise { + if (!intercepted) { + const state = await this.stat(); + if (state.dev === identity.dev && state.ino === identity.ino) { + intercepted = true; + if (physical === "cancel-after-read") { + controller.abort(abortReason); + return originalRead.apply(this, arguments_); + } + controller.abort(sameAsAbortReason ? readFailure : abortReason); + throw readFailure; + } + } + return originalRead.apply(this, arguments_); + }; + prototype.read = read; + try { + const result = await prepareReferenceSourceImport( + { + root, + caller: "reference-import-test", + policy: { secretPatterns: [] }, + signal: controller.signal, + }, + new ArtifactResourceScope(), + ); + expect(intercepted).toBe(true); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected ignore policy read failure"); + expect(result.error.code).toBe(expected); + expect(result.error.cause).toBe( + physical === "cancel-after-read" ? abortReason : readFailure, + ); + } finally { + prototype.read = originalRead; + } + }, + ); +}); + +describe("reference source ignore policy cleanup ownership", () => { + it("keeps a physical read primary when ignore-file cleanup remains pending", async () => { + const root = await createTestTempDirectory("rea-reference-ignore-cleanup-"); + const path = join(root, ".gitignore"); + await writeFile(path, "ignored/\n"); + const controller = new AbortController(); + const physical = Object.assign(new Error("controlled read failure"), { + code: "EIO", + }); + const probe = await open(path, "r"); + const identity = await probe.stat(); + const prototype = probe.constructor.prototype as { + read: (...arguments_: never[]) => Promise; + }; + await probe.close(); + const originalRead = prototype.read; + const ownerPrototype = OwnedFileHandle.prototype; + const originalClose = ownerPrototype.close; + let interceptedRead = false; + let closeFailures = 0; + prototype.read = async function ( + this: typeof probe, + ...arguments_: never[] + ): Promise { + const state = await this.stat(); + if ( + !interceptedRead && + state.dev === identity.dev && + state.ino === identity.ino + ) { + interceptedRead = true; + controller.abort(physical); + throw physical; + } + return originalRead.apply(this, arguments_); + }; + ownerPrototype.close = async function ( + this: OwnedFileHandle, + ): Promise { + const state = await this.handle.stat(); + if ( + state.dev === identity.dev && + state.ino === identity.ino && + closeFailures < 2 + ) { + closeFailures += 1; + throw new Error(`controlled close failure ${closeFailures}`); + } + return originalClose.call(this); + }; + const resources = new ArtifactResourceScope(); + try { + const result = await prepareReferenceSourceImport( + { + root, + caller: "reference-import-test", + policy: { secretPatterns: [] }, + signal: controller.signal, + }, + resources, + ); + expect(interceptedRead).toBe(true); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected ignore policy read failure"); + expect(result.error.code).toBe("io"); + expect(result.error.cause).toBe(physical); + expect(closeFailures).toBe(2); + expect(result.error.cleanup?.resources).toContain(path); + expect(resources.hasPending).toBe(true); + } finally { + await resources.retryCleanup(); + prototype.read = originalRead; + ownerPrototype.close = originalClose; + } + expect(resources.hasPending).toBe(false); + }); + + it("does not transfer cleanup ownership for a physical read shaped like a foreign cleanup failure", async () => { + const root = await createTestTempDirectory("rea-reference-ignore-opaque-"); + const path = join(root, ".gitignore"); + const foreignPath = join(root, "foreign.txt"); + await writeFile(path, "ignored/\n"); + await writeFile(foreignPath, "foreign bytes"); + const foreignOwner = new OwnedFileHandle(await open(foreignPath, "r")); + const reason = new RegularFileCleanupFailure( + foreignPath, + foreignOwner, + { kind: "completed", value: "foreign value" }, + new Error("foreign cleanup failure"), + ); + const controller = new AbortController(); + const probe = await open(path, "r"); + const identity = await probe.stat(); + const prototype = probe.constructor.prototype as { + read: (...arguments_: never[]) => Promise; + }; + await probe.close(); + const originalRead = prototype.read; + let intercepted = false; + prototype.read = async function ( + this: typeof probe, + ...arguments_: never[] + ): Promise { + const state = await this.stat(); + if ( + !intercepted && + state.dev === identity.dev && + state.ino === identity.ino + ) { + intercepted = true; + controller.abort(reason); + throw reason; + } + return originalRead.apply(this, arguments_); + }; + try { + const result = await prepareReferenceSourceImport( + { + root, + caller: "reference-import-test", + policy: { secretPatterns: [] }, + signal: controller.signal, + }, + new ArtifactResourceScope(), + ); + expect(intercepted).toBe(true); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected ignore policy read failure"); + expect(result.error.code).toBe("io"); + expect(result.error.cause).toBe(reason); + expect(foreignOwner.handle.fd).toBeGreaterThanOrEqual(0); + } finally { + prototype.read = originalRead; + await foreignOwner.close(); + } + }); +}); + describe("reference source symlink import", () => { it.each([ { target: "dist", reason: "default-ignored", pattern: "dist/" }, diff --git a/tests/support/failFileReadAfterAbort.ts b/tests/support/failFileReadAfterAbort.ts new file mode 100644 index 0000000000..2890dcaac5 --- /dev/null +++ b/tests/support/failFileReadAfterAbort.ts @@ -0,0 +1,40 @@ +import { open, type FileHandle } from "node:fs/promises"; +import { vi } from "vitest"; + +type PositionalFileRead = ( + this: FileHandle, + ...args: [Buffer, number, number, number | null] +) => Promise<{ readonly bytesRead: number; readonly buffer: Buffer }>; + +/** Fail one selected descriptor read after aborting with an independently chosen reason. */ +export const failFileReadAfterAbort = async ( + path: string, + controller: AbortController, + abortReason: unknown, + physicalCause: unknown, +): Promise => { + const probe = await open(path, "r"); + const identity = await probe.stat(); + const prototype = Object.getPrototypeOf(probe) as { + read: PositionalFileRead; + }; + await probe.close(); + const originalRead = prototype.read; + let failed = false; + vi.spyOn(prototype, "read").mockImplementation(async function ( + this: FileHandle, + ...args: [Buffer, number, number, number | null] + ) { + const current = await this.stat(); + if ( + !failed && + current.dev === identity.dev && + current.ino === identity.ino + ) { + failed = true; + controller.abort(abortReason); + throw physicalCause; + } + return originalRead.call(this, ...args); + }); +}; From 4852c830af1e3b9df437235b7a8fa8970a6d1c59 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:19:43 +0800 Subject: [PATCH 24/35] fix(native): preserve Mach-O names equal to version Parse only documented compound otool field labels so valid segment and section names are retained at the owning boundary. Verified real Apple clang fixtures through compiled CLI and stdio MCP without executing the targets. Closes #5680 --- src/native/parsers/otool.ts | 9 +++- .../native/nativeLoadKinds.golden.test.ts | 42 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/native/parsers/otool.ts b/src/native/parsers/otool.ts index 6da69d6a0d..ed2203f727 100644 --- a/src/native/parsers/otool.ts +++ b/src/native/parsers/otool.ts @@ -483,8 +483,13 @@ const parseField = (rawLine: string): readonly [string, string] | undefined => { const option = /^\s*(string #[1-9]\d*)(?: (.*))?$/u.exec(rawLine); if (option?.[1] !== undefined) return [option[1], option[2] ?? ""]; const line = rawLine.trim(); - // Match time stamp as a whole without swallowing `name stamp plugin.dylib`. - const match = /^((?:time stamp)|\S+(?:\s+version)?)(?:\s+(.+))?$/u.exec(line); + // These are the compound field labels used by otool. Keeping this list + // explicit prevents a one-word field value such as `sectname version` from + // being consumed as an invented `sectname version` key. + const match = + /^(time stamp|current version|compatibility version|\S+)(?:\s+(.+))?$/u.exec( + line, + ); return match?.[1] === undefined ? undefined : [match[1], match[2]?.trim() ?? ""]; diff --git a/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts b/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts index 84edc6b2ac..2c3e6d8903 100644 --- a/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts +++ b/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts @@ -165,6 +165,48 @@ it("retains the empty segment name of a Mach-O object file", () => { ).toThrow("otool field vmaddr has no value"); }); +it.each([ + { label: "segment", segmentName: "version", sectionName: "__mysect" }, + { label: "section", segmentName: "__DATA", sectionName: "version" }, +])( + "preserves a legal $label name equal to a compound field word", + ({ segmentName, sectionName }) => { + const output = [ + "Load command 0", + " cmd LC_SEGMENT_64", + " cmdsize 152", + ` segname ${segmentName}`, + " vmaddr 0x0000000000000000", + " vmsize 0x0000000000000018", + " fileoff 0", + " filesize 24", + " maxprot 0x00000007", + " initprot 0x00000007", + " nsects 1", + " flags 0x0", + "Section", + ` sectname ${sectionName}`, + ` segname ${segmentName}`, + " addr 0x0000000000000000", + " size 0x0000000000000018", + " offset 0", + " align 2^2 (4)", + " flags 0x80000400", + "", + ].join("\n"); + + const parsed = parseOtoolLoadCommands(withOtoolMachHeader(output)); + + expect(parsed.commands[0]?.fields.segname).toBe(segmentName); + expect(parsed.segments).toMatchObject([ + { + name: segmentName, + sections: [{ segment: segmentName, name: sectionName }], + }, + ]); + }, +); + it("preserves dependency install names that begin with the word stamp", () => { const load = parseOtoolLoadCommands( withOtoolMachHeader( From 52c84f6f58fa3cc7b39326654c6d3f261f957957 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:23:20 +0800 Subject: [PATCH 25/35] test(cli): bound ADB MCP validation to the advertised tool Reuse the exact selected advertised descriptor for SDK output validation instead of compiling the entire cached catalog for every fresh parity-test client. Retain actual catalog discovery, schema validation, and all result assertions under the existing worker heap limit. --- tests/boundary/cli/adbUninstallScope.test.ts | 8 ++++---- tests/boundary/cli/adbWindowDisplays.test.ts | 8 ++++---- tests/boundary/cli/adbWindowFocus.test.ts | 8 ++++---- tests/fixtures/advertisedMcpTool.ts | 8 +++++++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/boundary/cli/adbUninstallScope.test.ts b/tests/boundary/cli/adbUninstallScope.test.ts index ffa8683235..2ae0c295b3 100644 --- a/tests/boundary/cli/adbUninstallScope.test.ts +++ b/tests/boundary/cli/adbUninstallScope.test.ts @@ -60,10 +60,10 @@ if (args.length === 1 && args[0] === 'version') { harness.client, "uninstall_adb_package", ); - const response = await harness.client.callTool({ - name: schemas.name, - arguments: input, - }); + const response = await harness.client.callTool( + { name: schemas.name, arguments: input }, + { toolDefinition: schemas.toolDefinition }, + ); if (succeeds) { const evidence = parseEvidence(output.json); expect(evidence.normalized_result).toMatchObject({ diff --git a/tests/boundary/cli/adbWindowDisplays.test.ts b/tests/boundary/cli/adbWindowDisplays.test.ts index 0aa4f5c724..6718ed93f3 100644 --- a/tests/boundary/cli/adbWindowDisplays.test.ts +++ b/tests/boundary/cli/adbWindowDisplays.test.ts @@ -56,10 +56,10 @@ if (args.length === 1 && args[0] === 'version') { harness.client, "inspect_adb_window", ); - const response = await harness.client.callTool({ - name: schemas.name, - arguments: { serial }, - }); + const response = await harness.client.callTool( + { name: schemas.name, arguments: { serial } }, + { toolDefinition: schemas.toolDefinition }, + ); expect(response.isError).not.toBe(true); expect( schemas.ajv.validate(schemas.outputSchema, response.structuredContent), diff --git a/tests/boundary/cli/adbWindowFocus.test.ts b/tests/boundary/cli/adbWindowFocus.test.ts index d54163ceb7..4a0b89a44d 100644 --- a/tests/boundary/cli/adbWindowFocus.test.ts +++ b/tests/boundary/cli/adbWindowFocus.test.ts @@ -76,10 +76,10 @@ exit 1 ); const args = { serial }; expect(schemas.ajv.validate(schemas.inputSchema, args)).toBe(true); - const response = await harness.client.callTool({ - name: schemas.name, - arguments: args, - }); + const response = await harness.client.callTool( + { name: schemas.name, arguments: args }, + { toolDefinition: schemas.toolDefinition }, + ); expect(response.isError).not.toBe(true); expect( schemas.ajv.validate(schemas.outputSchema, response.structuredContent), diff --git a/tests/fixtures/advertisedMcpTool.ts b/tests/fixtures/advertisedMcpTool.ts index 854d272ba4..55521941de 100644 --- a/tests/fixtures/advertisedMcpTool.ts +++ b/tests/fixtures/advertisedMcpTool.ts @@ -20,7 +20,13 @@ export const advertisedMcpToolSchemas = async ( const outputSchema: Record = advertised.outputSchema; expect(ajv.validateSchema(inputSchema)).toBe(true); expect(ajv.validateSchema(outputSchema)).toBe(true); - return { name: advertised.name, ajv, inputSchema, outputSchema }; + return { + name: advertised.name, + toolDefinition: advertised, + ajv, + inputSchema, + outputSchema, + }; }; /** Keep public output, provenance and session retention checks together. */ From 56679dd2ae31dda58e181f8bf99775014f7bb82b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:45:34 +0800 Subject: [PATCH 26/35] fix(files): preserve physical read provenance through cleanup Keep raw physical read diagnostics alongside explicit operation provenance on the same cleanup owner. Preserve provenance after successful close retry for opted-in readers, restore legacy error projections, and prevent borrowed cleanup-shaped failures from authorizing foreign resource retries. Refs #5686 --- src/application/ClientConfigurationCleanup.ts | 5 +- src/application/RegularFileRead.ts | 44 +++-- src/filesystem/OwnedRegularFile.ts | 33 +++- .../regularFileCancellation.test.ts | 187 ++++++++++++++++++ 4 files changed, 256 insertions(+), 13 deletions(-) diff --git a/src/application/ClientConfigurationCleanup.ts b/src/application/ClientConfigurationCleanup.ts index bf0e310eab..596c32dcd9 100644 --- a/src/application/ClientConfigurationCleanup.ts +++ b/src/application/ClientConfigurationCleanup.ts @@ -74,7 +74,10 @@ export const retryClientConfigurationCleanup = async ( if (retried.cleanup !== undefined) observations.push(retried.cleanup); primary = retried.primary; const last = retried.failures.at(-1)!; - nested = last.outcome.kind === "failed" ? last.outcome.cause : undefined; + nested = + last.operationFailure === undefined && last.outcome.kind === "failed" + ? last.outcome.cause + : undefined; } else { const released = await resources.release({ kind: "lease", diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index 65ed2af261..31fc6bb432 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -12,6 +12,7 @@ import { RegularFileCancelled, regularFileCancellation, withOwnedRegularFile, + OwnedRegularFileOperationFailure, type RegularFileOperationOutcome, } from "../filesystem/OwnedRegularFile.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; @@ -29,12 +30,12 @@ export interface RegularFileReadOptions { readonly cancellationFailure?: (reason: unknown) => Error; } -/** A regular-file failure returned alongside an enabled cancellation factory. */ -export class RegularFileReadFailure extends Error { - readonly _tag = "RegularFileReadFailure"; +/** Preserves a physical descriptor-read cause until its owner has finalized. */ +export class RegularFileReadFailure extends OwnedRegularFileOperationFailure { + override readonly _tag = "RegularFileReadFailure"; - constructor(readonly operationCause: unknown) { - super("Regular file operation failed", { cause: operationCause }); + constructor(operationCause: unknown) { + super(operationCause); this.name = this._tag; } } @@ -65,6 +66,7 @@ export const retryRegularFileCleanup = async ( }> => { const outcome: RegularFileReadOutcome = failure.outcome.kind === "failed" && + failure.operationFailure === undefined && failure.outcome.cause instanceof RegularFileOperationCancelled && !(signal?.aborted === true && failure.outcome.cause === signal.reason) ? { kind: "failed", cause: failure.outcome.cause.reason } @@ -106,7 +108,10 @@ export const retryRegularFileCleanupFailures = async ( ? retried.outcome.cause : current.cleanupCause; const nested = - current.outcome.kind === "failed" ? current.outcome.cause : undefined; + current.operationFailure === undefined && + current.outcome.kind === "failed" + ? current.outcome.cause + : undefined; if (!(nested instanceof RegularFileCleanupFailure)) break; current = nested; } @@ -177,16 +182,27 @@ export const withRegularFile = async ( const retry = await retryRegularFileCleanup(cause, resources, signal); if (retry.cleanup !== undefined) throw cause; if (retry.outcome.kind === "failed") { - if (signal?.aborted === true && retry.outcome.cause === signal.reason) + if ( + cause.operationFailure === undefined && + signal?.aborted === true && + retry.outcome.cause === signal.reason + ) throw signal.reason; const cancellation = - regularFileCancellation(retry.outcome.cause) ?? - regularFileCancellation(cause); + cause.operationFailure === undefined + ? (regularFileCancellation(retry.outcome.cause) ?? + regularFileCancellation(cause)) + : undefined; if (cancellation !== undefined) { if (options.cancellationFailure !== undefined) throw options.cancellationFailure(cancellation.reason); throw cancellation.reason; } + if ( + cause.operationFailure !== undefined && + options.cancellationFailure !== undefined + ) + throw cause.operationFailure; throw retry.outcome.cause; } return retry.outcome.value; @@ -255,8 +271,8 @@ const readRegularFileContent = ( path: string, decode: (bytes: Buffer, state: Stats) => Value, options: RegularFileReadOptions, -): Promise => - withRegularFile( +): Promise => { + const operation = withRegularFile( path, async (handle, stats) => { try { @@ -282,3 +298,9 @@ const readRegularFileContent = ( }, options, ); + if (options.cancellationFailure !== undefined) return operation; + return operation.catch((cause: unknown) => { + if (cause instanceof RegularFileReadFailure) throw cause.operationCause; + throw cause; + }); +}; diff --git a/src/filesystem/OwnedRegularFile.ts b/src/filesystem/OwnedRegularFile.ts index aad2d1c1f9..25c0cac6f4 100644 --- a/src/filesystem/OwnedRegularFile.ts +++ b/src/filesystem/OwnedRegularFile.ts @@ -17,6 +17,16 @@ type CompletedRegularFileOutcome = Extract< { readonly kind: "completed" } >; +/** Physical operation origin kept separate from caller-provided error values. */ +export class OwnedRegularFileOperationFailure extends Error { + readonly _tag: string = "OwnedRegularFileOperationFailure"; + + constructor(readonly operationCause: unknown) { + super("Owned regular-file operation failed", { cause: operationCause }); + this.name = this._tag; + } +} + /** Cancellation after confirmed close, retaining the completed physical read. */ export class RegularFileCancelled extends Error { readonly _tag = "RegularFileCancelled"; @@ -54,6 +64,7 @@ export class RegularFileCleanupFailure extends Error { kind: "not_requested", }, readonly completionCancellation?: { readonly reason: unknown }, + readonly operationFailure?: OwnedRegularFileOperationFailure, ) { super(`Regular file cleanup failed for ${path}`, { cause: new AggregateError( @@ -78,6 +89,13 @@ export interface OwnedFileOperationOptions { export const regularFileCancellation = ( cause: unknown, ): { readonly reason: unknown } | undefined => { + if ( + cause instanceof RegularFileCleanupFailure && + cause.operationFailure !== undefined + ) + return cause.outcome.kind === "completed" + ? cause.completionCancellation + : undefined; const primary = cause instanceof RegularFileCleanupFailure && cause.outcome.kind === "failed" @@ -144,6 +162,18 @@ export const withOwnedFileHandle = async ( try { await owner.close(); } catch (cleanupCause: unknown) { + const operationFailure = + outcome.kind === "failed" && + outcome.cause instanceof OwnedRegularFileOperationFailure + ? outcome.cause + : undefined; + const retainedOutcome = + operationFailure === undefined + ? outcome + : { + kind: "failed" as const, + cause: operationFailure.operationCause, + }; const completionCancellation = outcome.kind === "completed" && options.signal?.aborted === true ? { reason: options.signal.reason } @@ -153,10 +183,11 @@ export const withOwnedFileHandle = async ( throw new RegularFileCleanupFailure( path, owner, - outcome, + retainedOutcome, cleanupCause, retainOwner(options.retainFailedClose, owner), completionCancellation, + operationFailure, ); } if (outcome.kind === "failed") throw outcome.cause; diff --git a/tests/boundary/filesystem/regularFileCancellation.test.ts b/tests/boundary/filesystem/regularFileCancellation.test.ts index 265d2f3f65..8e41becef8 100644 --- a/tests/boundary/filesystem/regularFileCancellation.test.ts +++ b/tests/boundary/filesystem/regularFileCancellation.test.ts @@ -5,7 +5,10 @@ import { expect, it, vi } from "vitest"; import { withRegularFile, + readRegularFile, + RegularFileReadFailure, retryRegularFileCleanup, + retryRegularFileCleanupFailures, } from "../../../src/application/RegularFileRead.js"; import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; import { RegularFileOperationCancelled } from "../../../src/filesystem/RegularFile.js"; @@ -88,6 +91,190 @@ it.each( }, ); +it("preserves a cleanup-shaped physical read rejection without borrowing its owner", async () => { + const root = await createTestTempDirectory( + "rea-file-physical-cleanup-shape-", + ); + const input = join(root, "input.txt"); + const foreignPath = join(root, "foreign.txt"); + await writeFile(input, "input"); + await writeFile(foreignPath, "foreign owner bytes"); + const foreignOwner = new OwnedFileHandle(await open(foreignPath, "r")); + const reason = new RegularFileCleanupFailure( + foreignPath, + foreignOwner, + { kind: "completed", value: "foreign value" }, + new Error("foreign cleanup failure"), + ); + const probe = await open(input, "r"); + const inputIdentity = await probe.stat(); + const prototype = probe.constructor.prototype as { + read: (...arguments_: never[]) => Promise; + }; + await probe.close(); + const originalRead = prototype.read; + prototype.read = async function ( + this: typeof probe, + ...arguments_: never[] + ): Promise { + const state = await this.stat(); + if (state.dev === inputIdentity.dev && state.ino === inputIdentity.ino) + throw reason; + return originalRead.apply(this, arguments_); + }; + try { + await expect(readRegularFile(input)).rejects.toBe(reason); + expect(foreignOwner.handle.fd).toBeGreaterThanOrEqual(0); + } finally { + prototype.read = originalRead; + await foreignOwner.close(); + } +}); + +it("retries only the owned close when a cleanup-shaped physical read remains pending", async () => { + const root = await createTestTempDirectory( + "rea-file-physical-cleanup-retry-", + ); + const input = join(root, "input.txt"); + const foreignPath = join(root, "foreign.txt"); + await writeFile(input, "input"); + await writeFile(foreignPath, "foreign owner bytes"); + const foreignOwner = new OwnedFileHandle(await open(foreignPath, "r")); + const reason = new RegularFileCleanupFailure( + foreignPath, + foreignOwner, + { kind: "completed", value: "foreign value" }, + new Error("foreign cleanup failure"), + ); + const controller = new AbortController(); + const probe = await open(input, "r"); + const inputIdentity = await probe.stat(); + const fileHandlePrototype = probe.constructor.prototype as { + read: (...arguments_: never[]) => Promise; + }; + await probe.close(); + const originalRead = fileHandlePrototype.read; + const ownerPrototype = OwnedFileHandle.prototype; + const originalClose = ownerPrototype.close; + let closeFailures = 0; + fileHandlePrototype.read = async function ( + this: typeof probe, + ...arguments_: never[] + ): Promise { + const state = await this.stat(); + if (state.dev === inputIdentity.dev && state.ino === inputIdentity.ino) { + controller.abort(reason); + throw reason; + } + return originalRead.apply(this, arguments_); + }; + ownerPrototype.close = async function (this: OwnedFileHandle): Promise { + const state = await this.handle.stat(); + if (state.dev === inputIdentity.dev && state.ino === inputIdentity.ino) { + closeFailures += 1; + if (closeFailures <= 2) + throw new Error(`owned close failure ${closeFailures}`); + } + return originalClose.call(this); + }; + const resources = new ArtifactResourceScope(); + try { + const failure = await readRegularFile(input, { + resources, + signal: controller.signal, + }).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(RegularFileCleanupFailure); + if (!(failure instanceof RegularFileCleanupFailure)) + throw new Error("Expected the opened input owner to remain retained"); + expect(failure.outcome).toMatchObject({ kind: "failed", cause: reason }); + expect(failure.operationFailure).toBeDefined(); + + const retried = await retryRegularFileCleanupFailures(failure, resources); + expect(retried.primary).toBe(reason); + expect(retried.failures).toEqual([failure]); + expect(retried.cleanup).toBeUndefined(); + expect(closeFailures).toBe(3); + expect(foreignOwner.handle.fd).toBeGreaterThanOrEqual(0); + } finally { + fileHandlePrototype.read = originalRead; + ownerPrototype.close = originalClose; + await resources.retryCleanup(); + await foreignOwner.close(); + } +}); + +it("retains physical read provenance for an opted-in caller after close retry", async () => { + const root = await createTestTempDirectory( + "rea-file-physical-cleanup-recovered-", + ); + const input = join(root, "input.txt"); + const foreignPath = join(root, "foreign.txt"); + await writeFile(input, "input"); + await writeFile(foreignPath, "foreign owner bytes"); + const foreignOwner = new OwnedFileHandle(await open(foreignPath, "r")); + const reason = new RegularFileCleanupFailure( + foreignPath, + foreignOwner, + { kind: "completed", value: "foreign value" }, + new Error("foreign cleanup failure"), + ); + const controller = new AbortController(); + const probe = await open(input, "r"); + const inputIdentity = await probe.stat(); + const fileHandlePrototype = probe.constructor.prototype as { + read: (...arguments_: never[]) => Promise; + }; + await probe.close(); + const originalRead = fileHandlePrototype.read; + const ownerPrototype = OwnedFileHandle.prototype; + const originalClose = ownerPrototype.close; + let closeFailures = 0; + let cancellationFactoryCalls = 0; + fileHandlePrototype.read = async function ( + this: typeof probe, + ...arguments_: never[] + ): Promise { + const state = await this.stat(); + if (state.dev === inputIdentity.dev && state.ino === inputIdentity.ino) { + controller.abort(reason); + throw reason; + } + return originalRead.apply(this, arguments_); + }; + ownerPrototype.close = async function (this: OwnedFileHandle): Promise { + const state = await this.handle.stat(); + if (state.dev === inputIdentity.dev && state.ino === inputIdentity.ino) { + closeFailures += 1; + if (closeFailures === 1) throw new Error("transient owned close failure"); + } + return originalClose.call(this); + }; + const resources = new ArtifactResourceScope(); + try { + const failure = await readRegularFile(input, { + resources, + signal: controller.signal, + cancellationFailure: (abortReason) => { + cancellationFactoryCalls += 1; + return new Error("cancelled", { cause: abortReason }); + }, + }).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(RegularFileReadFailure); + if (!(failure instanceof RegularFileReadFailure)) + throw new Error("Expected the physical read failure marker"); + expect(failure.operationCause).toBe(reason); + expect(cancellationFactoryCalls).toBe(0); + expect(closeFailures).toBe(2); + expect(resources.hasPending).toBe(false); + expect(foreignOwner.handle.fd).toBeGreaterThanOrEqual(0); + } finally { + fileHandlePrototype.read = originalRead; + ownerPrototype.close = originalClose; + await resources.retryCleanup(); + await foreignOwner.close(); + } +}); + it.each([false, true])( "preserves a nested read cancellation with the same signal=%s", async (sameSignal) => { From ccc7da309d7db385d5c6d73b0a3a8e093be33504 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:45:34 +0800 Subject: [PATCH 27/35] test(mcp): retain distinct Objective-C class observations Require both fixture addresses and unknown identity coverage when no selected-image binding proof permits coalescing. --- ...nhancedTools.01-lists-the-complete-target-open.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts index 7a14f1a1a5..a492fdf8e8 100644 --- a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts +++ b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts @@ -278,8 +278,12 @@ describe("enhanced MCP tools", () => { ), ); expect(results[0]).toMatchObject({ - count: 1, - classes: [{ address: "0x10", name: "_OBJC_CLASS_$_Fixture" }], + count: 2, + classes: [ + { address: "0x10", name: "_OBJC_CLASS_$_Fixture" }, + { address: "0x11", name: "_OBJC_CLASS_$_Fixture" }, + ], + identity_coverage: { status: "unknown" }, }); expect(results[1]).toMatchObject({ count: 1, From f789fd1fd8c6d1904b11ecbe8c71124291467d41 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:56:18 +0800 Subject: [PATCH 28/35] fix(native): count Swift vtable coverage in slot units Count examined and decoded vtable slots in the same population, preserving all returned slots and partial descriptor failures. Document the coverage unit and retain malformed-sibling and unresolved-slot checks. Closes #5681 --- docs/native-investigation.md | 2 ++ src/native/AppleDispatchMetadata.test.ts | 7 ++++++- src/native/AppleSwiftVtables.test.ts | 13 +++++++++---- src/native/AppleSwiftVtables.ts | 4 +++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 64417b7ed8..22cd354681 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -425,6 +425,8 @@ complete observation. vtable descriptors. Records include exact virtual addresses/file offsets and artifact evidence. Vtable indexes are metadata word offsets; witness indexes start after the conformance header. Names unavailable in metadata remain null. + The `swift_class_vtable_descriptors` coverage counters count examined vtable + slots and successfully decoded slots, rather than type-section entries. Pointers are decoded through the image's fixups: - `LC_DYLD_CHAINED_FIXUPS` rebases and binds, including the authenticated arm64e formats; diff --git a/src/native/AppleDispatchMetadata.test.ts b/src/native/AppleDispatchMetadata.test.ts index 5bff9c1388..6a46b9bac8 100644 --- a/src/native/AppleDispatchMetadata.test.ts +++ b/src/native/AppleDispatchMetadata.test.ts @@ -906,7 +906,12 @@ describe("malformed Apple dispatch metadata", () => { const coverage = result.coverage.find( ({ facet }) => facet === "swift_class_vtable_descriptors", ); - expect(coverage).toMatchObject({ status: "partial", examined: 3 }); + expect(coverage).toMatchObject({ + status: "partial", + examined: 2, + decoded: 2, + reason: expect.stringContaining("class name is empty"), + }); }); it("rejects malformed command and section boundaries", () => { diff --git a/src/native/AppleSwiftVtables.test.ts b/src/native/AppleSwiftVtables.test.ts index fe79b955e3..5873de9335 100644 --- a/src/native/AppleSwiftVtables.test.ts +++ b/src/native/AppleSwiftVtables.test.ts @@ -8,13 +8,13 @@ import type { DispatchRecordBudget } from "./AppleSwiftDispatchFacets.js"; import { issue } from "./AppleDispatchDecodeFacts.js"; describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => { - it("returns a located issue for an emitted unresolved async slot", () => { + it("counts examined slots and reports an unresolved async slot", () => { const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); const values = new Map([ [0x200n, 0x80000010], // class descriptor with a vtable [0x22cn, 0], // table offset - [0x230n, 1], // slot count - [0x21cn, 1], // descriptor positive words + [0x230n, 2], // slot count + [0x21cn, 2], // descriptor positive words [0x234n, 0x40], // async method flag ]); const facts = decodeSwiftClassVtables({ @@ -41,14 +41,17 @@ describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => { artifact_sha256: "a".repeat(64), }, ], - executable: () => false, + executable: (address) => address === 0x400n, admit: () => true, }, }); expect(result.swift_dispatch_slots[0]?.decode.status).toBe("partial"); + expect(result.swift_dispatch_slots[1]?.decode.status).toBe("decoded"); expect(facts).toMatchObject({ facet: "swift_class_vtable_descriptors", + examined: 2, + decoded: 1, exhaustive: false, issues: [{ code: "vtable_implementation_unresolved", location: "0x234" }], }); @@ -88,6 +91,8 @@ describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => { ), ).toMatchObject({ status: "partial", + examined: 2, + decoded: 1, reason: expect.stringContaining("0x234"), }); }); diff --git a/src/native/AppleSwiftVtables.ts b/src/native/AppleSwiftVtables.ts index 9a9c74308f..a680957ac2 100644 --- a/src/native/AppleSwiftVtables.ts +++ b/src/native/AppleSwiftVtables.ts @@ -25,6 +25,8 @@ export const decodeSwiftClassVtables = (input: { const { readers: read, result } = input; const failures: string[] = []; const issues = [] as ReturnType[]; + // Coverage counts vtable slots, matching `decoded`; descriptor entries are + // a discovery input and are not slot records. let examined = 0; let decoded = 0; for (const field of input.entries) { @@ -40,7 +42,6 @@ export const decodeSwiftClassVtables = (input: { ); break; } - examined++; try { const descriptor = read.relative(field); const flags = read.u32(descriptor); @@ -84,6 +85,7 @@ export const decodeSwiftClassVtables = (input: { ); break; } + examined++; const methodFlags = read.u32(entry); // Async slots reference async descriptors; coroutine and future kinds need separate decoders. const synchronous = From 9dee989a79536b962ac4b8694831fb2a04b429a4 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:01:17 +0800 Subject: [PATCH 29/35] fix(native): count Objective-C coverage by class object Count each admitted unique class or metaclass object at its parsing owner. Preserve emitted metadata, independent class-list scan bounds, partial failures, and truncation. Closes #5706 --- docs/native-investigation.md | 2 + src/native/AppleObjcClassCoverage.test.ts | 142 ++++++++++++++++++++++ src/native/AppleObjcDispatchFacets.ts | 8 +- 3 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 src/native/AppleObjcClassCoverage.test.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 22cd354681..f17f5bbc2f 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -427,6 +427,8 @@ complete observation. start after the conformance header. Names unavailable in metadata remain null. The `swift_class_vtable_descriptors` coverage counters count examined vtable slots and successfully decoded slots, rather than type-section entries. + The `objc_class_method_ivar_metadata` counters count admitted class and + metaclass objects and successfully decoded objects. Pointers are decoded through the image's fixups: - `LC_DYLD_CHAINED_FIXUPS` rebases and binds, including the authenticated arm64e formats; diff --git a/src/native/AppleObjcClassCoverage.test.ts b/src/native/AppleObjcClassCoverage.test.ts new file mode 100644 index 0000000000..eb48d13763 --- /dev/null +++ b/src/native/AppleObjcClassCoverage.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "vitest"; +import { objcRecordBudgetMacho } from "../../tests/fixtures/objcRecordBudgetMacho.js"; +import { decodeAppleDispatchMetadata } from "./AppleDispatchMetadata.js"; + +const base = 0x100000000n; +const provenance = { + path: "/fixtures/class-coverage.macho", + sha256: "a".repeat(64), +}; + +const classWithMetaclass = (options: { duplicateRoot?: boolean } = {}) => { + const bytes = objcRecordBudgetMacho("class"); + const pointer = (offset: number, target: number) => + bytes.writeBigUInt64LE(base + BigInt(target), offset); + + // Link the valid root class to a separate, equally valid metaclass object. + pointer(0x200, 0xa00); + pointer(0xa20, 0xa40); + bytes.writeUInt32LE(2, 0xa40); + pointer(0xa58, 0xb00); + bytes.write("FixtureClass\0", 0xb00); + + if (options.duplicateRoot) { + bytes.writeBigUInt64LE(16n, 144); + pointer(0x188, 0x200); + } + return bytes; +}; + +describe("Objective-C class-object coverage units", () => { + it("counts both admitted class and metaclass records", () => { + const result = decodeAppleDispatchMetadata( + classWithMetaclass(), + 100, + provenance, + ); + + expect(result.objc_classes).toMatchObject([ + { + name: "FixtureClass", + is_meta_class: false, + decode: { status: "decoded" }, + }, + { + name: "FixtureClass", + is_meta_class: true, + decode: { status: "decoded" }, + }, + ]); + expect( + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ), + ).toMatchObject({ status: "complete", examined: 2, decoded: 2 }); + }); + + it("counts a duplicate class-list identity once while retaining the two class objects", () => { + const result = decodeAppleDispatchMetadata( + classWithMetaclass({ duplicateRoot: true }), + 100, + provenance, + ); + + expect(result.objc_classes).toHaveLength(2); + expect( + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ), + ).toMatchObject({ status: "complete", examined: 2, decoded: 2 }); + }); + + it("keeps an admitted malformed class in examined and preserves its failure", () => { + const bytes = classWithMetaclass(); + bytes.writeUInt8(0, 0x900); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.objc_classes).toEqual([]); + expect( + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ), + ).toMatchObject({ + status: "partial", + examined: 1, + decoded: 0, + reason: expect.stringContaining("empty name"), + }); + }); + + it("preserves unknown-root partial facts for both admitted objects", () => { + const bytes = classWithMetaclass(); + bytes.writeUInt32LE(0, 0x300); + bytes.writeUInt32LE(0, 0xa40); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.objc_classes).toMatchObject([ + { + name: "FixtureClass", + is_meta_class: false, + super_class: null, + decode: { status: "partial", reason: "superclass_binding_unresolved" }, + }, + { + name: "FixtureClass", + is_meta_class: true, + super_class: null, + decode: { status: "partial", reason: "superclass_binding_unresolved" }, + }, + ]); + expect( + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ), + ).toMatchObject({ status: "partial", examined: 2, decoded: 0 }); + }); + + it("keeps object counts and truncation separate when the metaclass exceeds the budget", () => { + const result = decodeAppleDispatchMetadata( + classWithMetaclass(), + 2, + provenance, + ); + + expect(result.objc_classes).toMatchObject([ + { + name: "FixtureClass", + is_meta_class: false, + decode: { status: "decoded" }, + }, + ]); + expect( + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ), + ).toMatchObject({ + status: "partial", + examined: 1, + decoded: 1, + reason: expect.stringContaining("max_records_reached"), + }); + }); +}); diff --git a/src/native/AppleObjcDispatchFacets.ts b/src/native/AppleObjcDispatchFacets.ts index 5f507336e8..61397dbd91 100644 --- a/src/native/AppleObjcDispatchFacets.ts +++ b/src/native/AppleObjcDispatchFacets.ts @@ -97,7 +97,7 @@ export const decodeObjcDispatchFacets = (input: { } = input; const failures: string[] = []; const categoryIssues: DecodeIssue[] = []; - let examined = 0; + let classObjectsExamined = 0; const methods = ( list: bigint, className: string, @@ -273,6 +273,9 @@ export const decodeObjcDispatchFacets = (input: { const visited = new Set(); const readClass = (address: bigint, meta = false) => { if (address === 0n || visited.has(hex(address)) || !budget.admit()) return; + // Coverage counts admitted class objects, including recursively visited + // metaclasses, in the same units as the decoded objc_classes rows. + classObjectsExamined++; visited.add(hex(address)); const { ro, name } = readLocalClass(read, address); const root = (read.u32(ro) & 2) !== 0; @@ -419,7 +422,6 @@ export const decodeObjcDispatchFacets = (input: { } const field = section.address + BigInt(index * 8); const pointer = readClassListSlot(field); - if (pointer.ok || pointer.error === "unmapped") examined++; if (!pointer.ok) { if (pointer.error === "exhausted") { objcClassListScanTruncated = true; @@ -454,7 +456,7 @@ export const decodeObjcDispatchFacets = (input: { return { failures, categoryIssues, - examined, + examined: classObjectsExamined, categoriesExamined, objcListScanTruncated, objcClassListScanTruncated, From 0bd14ca16cd4e3be5a126daf709e1abe79c679f6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:16:47 +0800 Subject: [PATCH 30/35] fix(browser): preserve uncertain main-frame navigation identity Own CDP frame applicability and identity at the parsed event boundary. Preserve malformed or incomplete main-frame facts, recognize replacement roots, and carry the same document identity through captures, screenshots, timelines, and execution windows. Validation: full check:ci and docs:check; 763 browser and dispatch tests passed, 6 skipped. Independent source and fixture-lifecycle review passed. Local real-Chrome behavior was not exercised. Closes #5665 --- src/browser/CdpCaptureEventHandlers.ts | 40 +++- src/browser/CdpCaptureEventHelpers.test.ts | 102 +++++++++ src/browser/CdpCaptureEventHelpers.ts | 135 ++++++++++- src/browser/CdpCaptureEventState.ts | 1 + src/browser/CdpCaptureEvents.ts | 9 +- src/browser/CdpObservationSession.ts | 39 +++- src/browser/CdpPageCapture.ts | 4 + src/browser/CdpScreenshot.ts | 76 +++++-- src/browser/PlaywrightScenarioSession.test.ts | 57 +++-- .../execution/CdpExecutionWindow.test.ts | 54 +++++ src/browser/execution/CdpExecutionWindow.ts | 33 ++- ...erProvider.discovery_authorization.test.ts | 127 ++++++++++- .../browser/cdpBrowserProvider.support.ts | 14 +- .../browser/cdpObservationWindowScope.test.ts | 201 +++++++++++------ .../cdpScreenshotDocumentIdentity.test.ts | 210 +++++++++++++++--- tests/boundary/browser/cdpWebRuntime.test.ts | 39 ++++ 16 files changed, 962 insertions(+), 179 deletions(-) create mode 100644 src/browser/CdpCaptureEventHelpers.test.ts diff --git a/src/browser/CdpCaptureEventHandlers.ts b/src/browser/CdpCaptureEventHandlers.ts index b1b8b2ad67..493925d63f 100644 --- a/src/browser/CdpCaptureEventHandlers.ts +++ b/src/browser/CdpCaptureEventHandlers.ts @@ -25,7 +25,7 @@ import { initiatorLocation, integerOrNull, isJsonMediaType, - isMainFrameNavigation, + cdpFrameIdentity, } from "./CdpCaptureEventHelpers.js"; import type { CdpCaptureEventsState } from "./CdpCaptureEventState.js"; import { @@ -607,25 +607,47 @@ export const handleWebSocketCreated = ( export const handleFrameNavigated = ( state: CdpCaptureEventsState, - params: UnknownRecord, + params: unknown, ): void => { - const frame = recordValue(params.frame); if (state.mainFrameId === undefined) return; - if ( - !isMainFrameNavigation( - { method: "Page.frameNavigated", params }, - state.mainFrameId, + const identity = cdpFrameIdentity( + { method: "Page.frameNavigated", params }, + state.mainFrameId, + ); + if (identity.identityIssue !== null || identity.parentIssue !== null) { + state.completeness.exclude("frames", "invalid_protocol_value"); + if ( + identity.selection === "unknown" && + identity.parentRelation === "child" && + identity.identityIssue !== null ) - ) + return; + state.unknownRootIdentity = true; return; + } + if (identity.selection === "other" && identity.parentRelation === "child") + return; + if (identity.selection === "unknown") { + state.completeness.exclude("frames", "invalid_protocol_value"); + state.unknownRootIdentity = true; + return; + } + if (identity.selection === "other" && !identity.isRootReplacement) { + state.completeness.exclude("frames", "invalid_protocol_value"); + state.unknownRootIdentity = true; + return; + } + const frame = recordValue(recordValue(params)?.frame); const rawUrl = cdpStringValue(frame?.url); const loaderId = cdpStringValue(frame?.loaderId); // `frameNavigated` re-fires for a document already reported (same URL and // loader). Treating that as a navigation aborted a valid capture with // `target_changed`, which is a false claim that the target moved. - const document = `${String(loaderId ?? "")}\0${String(rawUrl ?? "")}`; + const document = `${String(identity.frameId ?? "")}\0${String(loaderId ?? "")}\0${String(rawUrl ?? "")}`; if (state.committedDocument === document) return; state.committedDocument = document; + if (identity.isRootReplacement) + state.mainFrameId = identity.frameId ?? undefined; state.navigationDuringCapture = true; if ( isHttpUrl(rawUrl) && diff --git a/src/browser/CdpCaptureEventHelpers.test.ts b/src/browser/CdpCaptureEventHelpers.test.ts new file mode 100644 index 0000000000..ad34274e45 --- /dev/null +++ b/src/browser/CdpCaptureEventHelpers.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { cdpFrameIdentity } from "./CdpCaptureEventHelpers.js"; + +describe("CDP frame identity ownership", () => { + it.each([ + ["missing", {}], + ["null", { frame: null }], + ["non-object", { frame: "frame" }], + ])("does not infer a root from a %s frame object", (_label, params) => { + expect( + cdpFrameIdentity( + { method: "Page.frameNavigated", params }, + "selected-main", + ), + ).toMatchObject({ + applicability: "applicable", + selection: "unknown", + isRootReplacement: false, + identityIssue: "missing_frame", + parentRelation: "unknown", + parentId: null, + parentIssue: null, + }); + }); + + it("recognizes a valid frame with no parent property as a root", () => { + expect( + cdpFrameIdentity( + { + method: "Page.frameNavigated", + params: { frame: { id: "replacement-main" } }, + }, + "selected-main", + ), + ).toMatchObject({ + applicability: "applicable", + selection: "other", + isRootReplacement: true, + identityIssue: null, + parentRelation: "root", + parentId: null, + parentIssue: null, + }); + }); + + it("does not interpret a malformed parent value as root evidence", () => { + expect( + cdpFrameIdentity( + { + method: "Page.frameNavigated", + params: { frame: { id: "replacement-main", parentId: "" } }, + }, + "selected-main", + ), + ).toMatchObject({ + selection: "other", + isRootReplacement: false, + identityIssue: null, + parentRelation: "unknown", + parentId: null, + parentIssue: "invalid_parent_id", + }); + }); + + it("preserves a self-parented frame ID but leaves its parent relation unknown", () => { + expect( + cdpFrameIdentity( + { + method: "Page.frameNavigated", + params: { + frame: { id: "candidate-root", parentId: "candidate-root" }, + }, + }, + "selected-main", + ), + ).toMatchObject({ + selection: "other", + isRootReplacement: false, + frameId: "candidate-root", + identityIssue: null, + parentRelation: "unknown", + parentId: null, + parentIssue: "self_parent", + }); + }); + + it("retains the selected-main parent issue for a self-parented selected frame", () => { + expect( + cdpFrameIdentity( + { + method: "Page.frameNavigated", + params: { frame: { id: "selected-main", parentId: "selected-main" } }, + }, + "selected-main", + ), + ).toMatchObject({ + selection: "main", + parentRelation: "unknown", + parentIssue: "main_frame_has_parent", + }); + }); +}); diff --git a/src/browser/CdpCaptureEventHelpers.ts b/src/browser/CdpCaptureEventHelpers.ts index 6f718f0d6c..d3e60f2450 100644 --- a/src/browser/CdpCaptureEventHelpers.ts +++ b/src/browser/CdpCaptureEventHelpers.ts @@ -8,21 +8,132 @@ import { } from "./CdpCaptureValues.js"; import type { CdpEvent } from "./CdpConnection.js"; -/** Identify a top-level navigation event from its untrusted CDP envelope. */ -export const isMainFrameNavigation = ( +export type FrameSelection = "main" | "other" | "unknown"; +export type FrameParentRelation = + | "root" + | "child" + | "unknown" + | "not_applicable"; + +/** Normalized frame identity facts shared by all selected-frame consumers. */ +export interface CdpFrameIdentity { + readonly applicability: "applicable" | "not_applicable"; + readonly selection: FrameSelection; + readonly isRootReplacement: boolean; + readonly frameId: string | null; + readonly identityIssue: + | "missing_params" + | "invalid_params" + | "missing_frame" + | "missing_id" + | "invalid_id" + | null; + readonly parentRelation: FrameParentRelation; + readonly parentId: string | null; + readonly parentIssue: + | "invalid_parent_id" + | "self_parent" + | "main_frame_has_parent" + | null; +} + +const nonEmptyString = (value: unknown): string | undefined => + typeof value === "string" && value.length > 0 ? value : undefined; + +const identityIssueFor = ( + navigated: boolean, + frame: UnknownRecord | undefined, + rawId: unknown, +): CdpFrameIdentity["identityIssue"] => { + if (navigated && frame === undefined) return "missing_frame"; + if (rawId === undefined) return "missing_id"; + return nonEmptyString(rawId) === undefined ? "invalid_id" : null; +}; + +const parentFacts = ( + navigated: boolean, + frame: UnknownRecord | undefined, +): Pick => { + if (!navigated) + return { + parentRelation: "not_applicable", + parentId: null, + parentIssue: null, + }; + if (frame === undefined) + return { parentRelation: "unknown", parentId: null, parentIssue: null }; + if (!("parentId" in frame)) + return { parentRelation: "root", parentId: null, parentIssue: null }; + const parentId = nonEmptyString(frame.parentId); + return parentId === undefined || parentId === nonEmptyString(frame.id) + ? { + parentRelation: "unknown", + parentId: null, + parentIssue: + parentId === undefined ? "invalid_parent_id" : "self_parent", + } + : { parentRelation: "child", parentId, parentIssue: null }; +}; + +/** Parse required frame identity separately from the optional parent fact. */ +export const cdpFrameIdentity = ( event: CdpEvent, mainFrameId: string, -): boolean => { +): CdpFrameIdentity => { + const navigated = event.method === "Page.frameNavigated"; + const requested = event.method === "Page.frameRequestedNavigation"; + const sameDocument = event.method === "Page.navigatedWithinDocument"; + if (!navigated && !requested && !sameDocument) + return { + applicability: "not_applicable", + selection: "unknown", + isRootReplacement: false, + frameId: null, + identityIssue: null, + parentRelation: "not_applicable", + parentId: null, + parentIssue: null, + }; const params = recordValue(event.params); - if (params === undefined) return false; - const frameId = - event.method === "Page.frameNavigated" - ? cdpStringValue(recordValue(params.frame)?.id) - : event.method === "Page.frameRequestedNavigation" || - event.method === "Page.navigatedWithinDocument" - ? cdpStringValue(params.frameId) - : undefined; - return frameId === mainFrameId; + if (params === undefined) + return { + applicability: "applicable", + selection: "unknown", + isRootReplacement: false, + frameId: null, + identityIssue: + event.params === undefined ? "missing_params" : "invalid_params", + parentRelation: "not_applicable", + parentId: null, + parentIssue: null, + }; + const frame = navigated ? recordValue(params.frame) : undefined; + const rawId = navigated ? frame?.id : params.frameId; + const frameId = nonEmptyString(rawId); + const selection = + frameId === undefined + ? "unknown" + : frameId === mainFrameId + ? "main" + : "other"; + const identityIssue = identityIssueFor(navigated, frame, rawId); + const parent = parentFacts(navigated, frame); + return { + applicability: "applicable", + selection, + isRootReplacement: + selection === "other" && parent.parentRelation === "root", + frameId: frameId ?? null, + identityIssue, + parentRelation: parent.parentRelation, + parentId: parent.parentId, + parentIssue: + selection === "main" && + (parent.parentRelation === "child" || + parent.parentIssue === "self_parent") + ? "main_frame_has_parent" + : parent.parentIssue, + }; }; export const firstCallFrame = ( diff --git a/src/browser/CdpCaptureEventState.ts b/src/browser/CdpCaptureEventState.ts index 9883a3a3c1..2e72bd37ce 100644 --- a/src/browser/CdpCaptureEventState.ts +++ b/src/browser/CdpCaptureEventState.ts @@ -26,6 +26,7 @@ export interface CdpCaptureEventsState { readonly completeness: CdpCaptureCompleteness; originViolation: boolean; navigationDuringCapture: boolean; + unknownRootIdentity: boolean; mainFrameId: string | undefined; /** * Identity of the document last observed in the main frame. `frameNavigated` diff --git a/src/browser/CdpCaptureEvents.ts b/src/browser/CdpCaptureEvents.ts index b3f1ce41b8..96b8bba7b7 100644 --- a/src/browser/CdpCaptureEvents.ts +++ b/src/browser/CdpCaptureEvents.ts @@ -45,6 +45,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { ]); originViolation = false; navigationDuringCapture = false; + unknownRootIdentity = false; mainFrameId: string | undefined = undefined; committedDocument: string | undefined = undefined; @@ -60,6 +61,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { this.mainFrameId = frameId; this.originViolation = false; this.navigationDuringCapture = false; + this.unknownRootIdentity = false; this.committedDocument = undefined; this.scripts.clear(); this.scriptMetadataBudgetOmissionsById.clear(); @@ -122,6 +124,10 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { } ingest(event: CdpEvent): void { + if (event.method === "Page.frameNavigated") { + handlers.handleFrameNavigated(this, event.params); + return; + } const params = recordValue(event.params); if (params === undefined) return; switch (event.method) { @@ -158,9 +164,6 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { case "Runtime.consoleAPICalled": handlers.handleConsoleAPICalled(this, params); break; - case "Page.frameNavigated": - handlers.handleFrameNavigated(this, params); - break; case "Network.webSocketCreated": handlers.handleWebSocketCreated(this, params); break; diff --git a/src/browser/CdpObservationSession.ts b/src/browser/CdpObservationSession.ts index 3500a3444d..51adfe5612 100644 --- a/src/browser/CdpObservationSession.ts +++ b/src/browser/CdpObservationSession.ts @@ -25,7 +25,7 @@ import { type UnknownRecord, } from "./CdpCaptureValues.js"; import { mainFrameUrl } from "./CdpCaptureDocuments.js"; -import { isMainFrameNavigation } from "./CdpCaptureEventHelpers.js"; +import { cdpFrameIdentity } from "./CdpCaptureEventHelpers.js"; import { cdpTargetEventMatches } from "./CdpTargetEvents.js"; interface ObservationContext { @@ -184,7 +184,7 @@ class TimelineCapture { constructor( private readonly allowedOrigins: ReadonlySet, - private readonly mainFrameId: string, + private mainFrameId: string, initialUrl: string, ) { this.finalUrl = @@ -204,7 +204,15 @@ class TimelineCapture { #ingestArmed(event: CdpTransportEvent, bufferedBeforeArm = false): void { if (this.#endReason !== undefined || this.#scopeExcluded) return; const params = recordValue(event.params); - if (params === undefined) return; + if (params === undefined) { + if ( + event.method === "Page.frameRequestedNavigation" || + event.method === "Page.frameNavigated" || + event.method === "Page.navigatedWithinDocument" + ) + this.completeness.exclude("timeline", "invalid_protocol_value"); + return; + } const contextOnly = bufferedBeforeArm && hasNoComparableProducerTimestamp(event.method); if (contextOnly) this.#hasUnknownEventTime = true; @@ -326,7 +334,10 @@ class TimelineCapture { params: UnknownRecord, contextOnly: boolean, ): void { - if (!isMainFrameNavigation(event, this.mainFrameId)) return; + const identity = cdpFrameIdentity(event, this.mainFrameId); + if (identity.identityIssue !== null || identity.parentIssue !== null) + this.completeness.exclude("timeline", "invalid_protocol_value"); + if (identity.selection !== "main") return; const reason = safeDetail(params.reason); const rawUrl = cdpStringValue(params.url); const destination = @@ -349,12 +360,23 @@ class TimelineCapture { contextOnly: boolean, ): void { const frame = recordValue(params.frame); - if (!isMainFrameNavigation(event, this.mainFrameId)) return; + const identity = cdpFrameIdentity(event, this.mainFrameId); + if (identity.identityIssue !== null || identity.parentIssue !== null) + this.completeness.exclude("timeline", "invalid_protocol_value"); + if (identity.selection !== "main" && !identity.isRootReplacement) return; const rawUrl = cdpStringValue(frame?.url); const destination = scopedUrl(rawUrl, this.allowedOrigins); + if (identity.isRootReplacement) { + this.mainFrameId = identity.frameId ?? this.mainFrameId; + this.#pendingReload = false; + if (contextOnly) { + this.#stopIfOutOfScope(destination); + return; + } + } this.#add({ type: - !contextOnly && this.#pendingReload + !identity.isRootReplacement && !contextOnly && this.#pendingReload ? "same_origin_reload" : "navigation_committed", params: { ...params, frameId: frame?.id, loaderId: frame?.loaderId }, @@ -374,7 +396,10 @@ class TimelineCapture { params: UnknownRecord, contextOnly: boolean, ): void { - if (!isMainFrameNavigation(event, this.mainFrameId)) return; + const identity = cdpFrameIdentity(event, this.mainFrameId); + if (identity.identityIssue !== null || identity.parentIssue !== null) + this.completeness.exclude("timeline", "invalid_protocol_value"); + if (identity.selection !== "main") return; const rawUrl = cdpStringValue(params.url); const destination = scopedUrl(rawUrl, this.allowedOrigins); this.#add({ diff --git a/src/browser/CdpPageCapture.ts b/src/browser/CdpPageCapture.ts index a986366cb0..ba3eb00ef5 100644 --- a/src/browser/CdpPageCapture.ts +++ b/src/browser/CdpPageCapture.ts @@ -170,6 +170,8 @@ const captureAuthorizedPage = async ( operation: context.operation, }); const completedUrl = mainFrameUrl(completedFrameResult) ?? ""; + if (state.events.unknownRootIdentity) + throw new BrowserObservationError("inspect_web_page", "protocol_error"); if (state.events.originViolation) throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); if (state.events.navigationDuringCapture || completedUrl !== attachedUrl) @@ -264,6 +266,8 @@ const authorizeObservationWindow = async ( context.operation, signal, ); + if (events.unknownRootIdentity) + throw new BrowserObservationError("inspect_web_page", "protocol_error"); if (events.originViolation) throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); }; diff --git a/src/browser/CdpScreenshot.ts b/src/browser/CdpScreenshot.ts index e4093479b1..59a9e7768d 100644 --- a/src/browser/CdpScreenshot.ts +++ b/src/browser/CdpScreenshot.ts @@ -5,7 +5,7 @@ import type { } from "../domain/webScreenshot.js"; import { createWebScreenshotArtifact } from "../domain/webScreenshot.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; -import { isMainFrameNavigation } from "./CdpCaptureEventHelpers.js"; +import { cdpFrameIdentity } from "./CdpCaptureEventHelpers.js"; import type { CdpEndpointDiscovery, CdpEndpointTarget } from "./CdpEndpoint.js"; import type { CdpConnection, CdpEvent } from "./CdpConnection.js"; import { @@ -63,12 +63,15 @@ export const captureCdpScreenshot = async ( const navigation = { changed: false, leftScope: false, + invalidIdentity: false, + frameId: mainFrame.frame_id, url: beforeUrl, loaderId: cdpStringValue(initialFrame?.loaderId), + completeness: new CdpCaptureCompleteness(), }; const removeListener = context.connection.onEvent((event) => { if (event.sessionId !== context.sessionId) return; - observeScreenshotNavigation(event, mainFrame.frame_id, origins, navigation); + observeScreenshotNavigation(event, origins, navigation); }); let result: ReturnType; let after: unknown; @@ -90,13 +93,10 @@ export const captureCdpScreenshot = async ( } finally { removeListener(); } - const encoded = cdpStringValue(result?.data); - const bytes = - encoded === undefined ? undefined : decodeCanonicalBase64(encoded); - if (bytes === undefined) + const screenshot = decodeScreenshot(result); + if (screenshot === undefined) throw new BrowserObservationError("inspect_web_page", "protocol_error"); - const dimensions = readPngHeaderDimensions(bytes); - if (dimensions === undefined) + if (navigation.invalidIdentity) throw new BrowserObservationError("inspect_web_page", "protocol_error"); if (navigation.leftScope) throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); @@ -122,9 +122,9 @@ export const captureCdpScreenshot = async ( origin: authorized.origin ?? "", }, captured_at: new Date().toISOString(), - viewport: dimensions, - artifact: createWebScreenshotArtifact(bytes), - completeness: new CdpCaptureCompleteness().snapshot(), + viewport: screenshot.dimensions, + artifact: createWebScreenshotArtifact(screenshot.bytes), + completeness: navigation.completeness.snapshot(), limitations: [ "The screenshot contains the visible viewport, including on-screen content.", "REA does not scroll, evaluate JavaScript, or capture beyond the current viewport.", @@ -132,24 +132,62 @@ export const captureCdpScreenshot = async ( }; }; +const decodeScreenshot = ( + result: ReturnType, +): + | { + readonly bytes: Buffer; + readonly dimensions: { width: number; height: number }; + } + | undefined => { + const encoded = cdpStringValue(result?.data); + const bytes = + encoded === undefined ? undefined : decodeCanonicalBase64(encoded); + if (bytes === undefined) return undefined; + const dimensions = readPngHeaderDimensions(bytes); + return dimensions === undefined ? undefined : { bytes, dimensions }; +}; + const observeScreenshotNavigation = ( event: CdpEvent, - mainFrameId: string, origins: ReadonlySet, state: { changed: boolean; leftScope: boolean; - readonly url: string | undefined; - readonly loaderId: string | undefined; + invalidIdentity: boolean; + frameId: string; + url: string | undefined; + loaderId: string | undefined; + readonly completeness: CdpCaptureCompleteness; }, ): void => { - if (!isMainFrameNavigation(event, mainFrameId)) return; + const identity = cdpFrameIdentity(event, state.frameId); + if (identity.applicability === "not_applicable") return; + if (identity.identityIssue !== null || identity.parentIssue !== null) { + state.completeness.exclude("screenshot", "invalid_protocol_value"); + if ( + identity.selection === "unknown" && + identity.parentRelation === "child" && + identity.identityIssue !== null + ) + return; + state.invalidIdentity = true; + return; + } + if (identity.selection === "unknown") { + if (identity.parentRelation === "child") return; + state.completeness.exclude("screenshot", "invalid_protocol_value"); + state.invalidIdentity = true; + return; + } + if (identity.selection === "other" && !identity.isRootReplacement) return; const params = recordValue(event.params); if (params === undefined) return; const frame = event.method === "Page.frameNavigated" ? recordValue(params.frame) : params; if ( event.method === "Page.frameNavigated" && + !identity.isRootReplacement && state.loaderId !== undefined && state.loaderId !== "" && cdpStringValue(frame?.loaderId) === state.loaderId && @@ -160,4 +198,12 @@ const observeScreenshotNavigation = ( state.changed = true; if (allowedSanitizedUrl(frame?.url, origins) === undefined) state.leftScope = true; + if (event.method === "Page.frameNavigated") { + if (identity.isRootReplacement) + state.frameId = identity.frameId ?? state.frameId; + state.url = cdpStringValue(frame?.url) ?? state.url; + state.loaderId = cdpStringValue(frame?.loaderId); + } else if (event.method === "Page.navigatedWithinDocument") { + state.url = cdpStringValue(frame?.url) ?? state.url; + } }; diff --git a/src/browser/PlaywrightScenarioSession.test.ts b/src/browser/PlaywrightScenarioSession.test.ts index 7615c27dd6..62c3f3e179 100644 --- a/src/browser/PlaywrightScenarioSession.test.ts +++ b/src/browser/PlaywrightScenarioSession.test.ts @@ -216,7 +216,7 @@ const policyResponse = (method: string): unknown => ? { result: { value: null } } : {}; -const prepareLateStopRetry = async () => { +const prepareHeldPolicyRestoration = async () => { const { cleanup, page, storage } = fixture(); const cancellation = new AbortController(); storage?.close.mockResolvedValue({ loadingStopped: false }); @@ -241,6 +241,28 @@ const prepareLateStopRetry = async () => { return policyResponse(method); }); + return { + cleanup, + page, + storage, + cancellation, + session, + finishRestoration, + started, + transportClosed, + }; +}; + +const prepareLateStopRetry = async () => { + const { + cleanup, + page, + cancellation, + session, + finishRestoration, + started, + transportClosed, + } = await prepareHeldPolicyRestoration(); vi.useFakeTimers(); try { const firstClose = session.close(false).then( @@ -417,29 +439,16 @@ it("transfers a late stop request after storage has already released", async () it.each(["pending", "settled"] as const)( "honors a late stop after policy restoration is %s", async (phase) => { - const { cleanup, page, storage } = fixture(); - const cancellation = new AbortController(); - storage?.close.mockResolvedValue({ loadingStopped: false }); - const attached = connectedScenario("block"); - const session = await PlaywrightScenarioSession.open( - attached, - {}, - { signal: cancellation.signal }, - ); - const { promise: held, resolve: finishRestoration } = deferred(); - const { promise: started, resolve: restorationStarted } = deferred(); - const { promise: transportClosed, resolve: signalTransportClosed } = - deferred(); - mocks.policyTransportClose.mockImplementation(async () => { - signalTransportClosed(); - }); - mocks.policySend.mockImplementation(async (method: string) => { - if (method === "Runtime.evaluate") { - restorationStarted(); - await held; - } - return policyResponse(method); - }); + const { + cleanup, + page, + storage, + cancellation, + session, + finishRestoration, + started, + transportClosed, + } = await prepareHeldPolicyRestoration(); vi.useFakeTimers(); try { diff --git a/src/browser/execution/CdpExecutionWindow.test.ts b/src/browser/execution/CdpExecutionWindow.test.ts index e906be92ab..4a6268148d 100644 --- a/src/browser/execution/CdpExecutionWindow.test.ts +++ b/src/browser/execution/CdpExecutionWindow.test.ts @@ -63,3 +63,57 @@ it("settles caller cancellation and removes its local timer", async () => { vi.useRealTimers(); } }); + +it("keeps a root document transition distinct from a child frame event", async () => { + const root = new CdpExecutionWindow({ + target: { frame_id: "selected-main" }, + operation: "observe_web_execution", + options: {}, + }); + const rootResult = root.start(30_000); + root.ingest({ + method: "Page.frameNavigated", + params: { frame: { id: "new-main", url: "https://example.test/" } }, + }); + await expect(rootResult).resolves.toBe("document_changed"); + + const child = new CdpExecutionWindow({ + target: { frame_id: "selected-main" }, + operation: "observe_web_execution", + options: {}, + }); + const childResult = child.start(30_000); + child.ingest({ + method: "Page.frameNavigated", + params: { + frame: { + id: "child-frame", + parentId: "selected-main", + url: "https://example.test/child", + }, + }, + }); + expect(child.active).toBe(true); + child.end("window_elapsed"); + await expect(childResult).resolves.toBe("window_elapsed"); +}); + +it("surfaces malformed frame identity as a protocol failure without guessing a transition", async () => { + const window = new CdpExecutionWindow({ + target: { frame_id: "selected-main" }, + operation: "observe_web_execution", + options: {}, + }); + const result = window.start(30_000); + window.ingest({ + method: "Page.frameNavigated", + params: { + frame: { id: "child-frame", parentId: 7, url: "https://example.test/" }, + }, + }); + await expect(result).rejects.toMatchObject({ + _tag: "BrowserObservationError", + reason: "protocol_error", + }); + expect(window.reason).toBeUndefined(); +}); diff --git a/src/browser/execution/CdpExecutionWindow.ts b/src/browser/execution/CdpExecutionWindow.ts index f8fa4b7eff..fddfe10329 100644 --- a/src/browser/execution/CdpExecutionWindow.ts +++ b/src/browser/execution/CdpExecutionWindow.ts @@ -3,6 +3,8 @@ import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { WebExecution } from "../../domain/webExecution.js"; import type { CdpEvent } from "../CdpConnection.js"; import type { CdpRuntimeSession } from "./CdpRuntimeSession.js"; +import { BrowserObservationError } from "../../domain/browserObservationError.js"; +import { cdpFrameIdentity } from "../CdpCaptureEventHelpers.js"; type EndReason = WebExecution["window"]["end_reason"]; /** One finite document-scoped window, with an actual armed point and event-driven termination. */ @@ -33,15 +35,30 @@ export class CdpExecutionWindow { if (!this.active) return; if (event.method === "Runtime.executionContextsCleared") this.end("document_changed"); - if ( - event.method === "Page.frameNavigated" && - typeof event.params === "object" && - event.params !== null && - "frame" in event.params - ) { - const frame = event.params.frame; - if (typeof frame === "object" && frame !== null && !("parentId" in frame)) + if (event.method === "Page.frameNavigated") { + const identity = cdpFrameIdentity(event, this.session.target.frame_id); + if (identity.identityIssue !== null || identity.parentIssue !== null) { + this.fail( + new BrowserObservationError(this.session.operation, "protocol_error"), + ); + return; + } + if (identity.selection === "main") { this.end("document_changed"); + return; + } + if (identity.parentRelation === "child") return; + if ( + identity.selection === "other" && + identity.parentRelation === "root" + ) { + this.end("document_changed"); + return; + } + this.fail( + new BrowserObservationError(this.session.operation, "protocol_error"), + ); + return; } if ( event.method === "Page.frameDetached" && diff --git a/tests/boundary/browser/cdpBrowserProvider.discovery_authorization.test.ts b/tests/boundary/browser/cdpBrowserProvider.discovery_authorization.test.ts index 053cd673ea..3f17c1d813 100644 --- a/tests/boundary/browser/cdpBrowserProvider.discovery_authorization.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.discovery_authorization.test.ts @@ -5,8 +5,30 @@ import { inspectWebPageInputSchema, listBrowserTargetsInputSchema, } from "../../../src/domain/browserObservation.js"; -import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; -import { trackBrowser } from "./cdpBrowserProvider.support.js"; +import { + startFakeCdpBrowser, + type FakeOptions, +} from "../../fixtures/fakeCdpBrowser.js"; +import { frameNavigated, trackBrowser } from "./cdpBrowserProvider.support.js"; + +const inspectWithNetworkEvents = async ( + commandEvents: NonNullable, +) => { + const browser = await startFakeCdpBrowser({ + eventsBeforeReply: ["Network.enable"], + commandEvents, + }); + trackBrowser(browser); + const result = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 1, + }), + ); + return { browser, result }; +}; describe("CdpBrowserProvider: discovery authorization 1", () => { it("discovers pages and captures a selected target without an origin list", async () => { @@ -158,6 +180,107 @@ describe("CdpBrowserProvider: discovery authorization 1", () => { }); }); +it("fails with a protocol error after an unidentifiable frame event and detaches", async () => { + const { browser, result } = await inspectWithNetworkEvents((command) => + command.method === "Network.enable" + ? [ + { + method: "Page.frameNavigated", + ...(command.sessionId === undefined + ? {} + : { sessionId: command.sessionId }), + params: undefined, + }, + ] + : undefined, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "protocol_error" }, + }); + await expect.poll(() => browser.activeConnectionCount()).toBe(0); +}); + +it("retains malformed child-frame identity as frame coverage while completing capture", async () => { + const { browser, result } = await inspectWithNetworkEvents((command) => + command.method === "Network.enable" + ? [ + frameNavigated(command, { + id: 3, + parentId: "frame-main", + url: "https://outside.invalid/child", + }), + ] + : undefined, + ); + if (!result.ok) throw result.error; + expect(result.value.completeness.unavailable_sections).toContain("frames"); + expect(result.value.target.url).toBe( + `${browser.allowedOrigin}/app?token=frame-secret`, + ); +}); + +it("fails capture on a contradictory self-parented frame instead of accepting it as a child", async () => { + const { browser, result } = await inspectWithNetworkEvents( + (command, origin) => + command.method === "Network.enable" + ? [ + frameNavigated(command, { + id: "candidate-root", + parentId: "candidate-root", + url: `${origin}/app?token=frame-secret`, + }), + ] + : undefined, + ); + expect(result).toMatchObject({ + ok: false, + error: { reason: "protocol_error" }, + }); + await expect.poll(() => browser.activeConnectionCount()).toBe(0); +}); + +it("tracks a known root replacement through later frame-scope events", async () => { + const { result } = await inspectWithNetworkEvents((command, origin) => + command.method === "Network.enable" + ? [ + frameNavigated(command, { + id: "replacement-main", + loaderId: "replacement-loader", + url: `${origin}/app?token=frame-secret`, + }), + frameNavigated(command, { + id: "replacement-main", + loaderId: "outside-loader", + url: "https://outside.invalid/path", + }), + ] + : undefined, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "target_not_allowed" }, + }); +}); + +it("treats a same-URL and same-loader root frame ID change as a capture change", async () => { + const { result } = await inspectWithNetworkEvents((command, origin) => + command.method === "Network.enable" + ? [ + frameNavigated(command, { + id: "replacement-main", + loaderId: "loader-main", + url: `${origin}/app?token=frame-secret`, + }), + ] + : undefined, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "target_changed" }, + }); +}); + describe("CdpBrowserProvider: discovery authorization 2", () => { it("rejects empty browser attachment session identifiers", async () => { const browser = await startFakeCdpBrowser({ invalidAttachedSession: true }); diff --git a/tests/boundary/browser/cdpBrowserProvider.support.ts b/tests/boundary/browser/cdpBrowserProvider.support.ts index ebb688df55..d1d2357191 100644 --- a/tests/boundary/browser/cdpBrowserProvider.support.ts +++ b/tests/boundary/browser/cdpBrowserProvider.support.ts @@ -1,6 +1,9 @@ import { afterEach } from "vitest"; -import type { FakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; +import type { + FakeCdpBrowser, + FakeOptions, +} from "../../fixtures/fakeCdpBrowser.js"; const browsers: FakeCdpBrowser[] = []; @@ -12,6 +15,15 @@ export const trackBrowser = (browser: FakeCdpBrowser): void => { browsers.push(browser); }; +export const frameNavigated = ( + command: Parameters>[0], + frame: unknown, +) => ({ + method: "Page.frameNavigated", + ...(command.sessionId === undefined ? {} : { sessionId: command.sessionId }), + params: { frame }, +}); + export const waitFor = async (ready: () => boolean, description: string) => { const deadline = Date.now() + 5_000; while (!ready()) { diff --git a/tests/boundary/browser/cdpObservationWindowScope.test.ts b/tests/boundary/browser/cdpObservationWindowScope.test.ts index c3b8e5f10c..7e76b967c8 100644 --- a/tests/boundary/browser/cdpObservationWindowScope.test.ts +++ b/tests/boundary/browser/cdpObservationWindowScope.test.ts @@ -6,12 +6,12 @@ import { startFakeCdpBrowser, type FakeOptions, } from "../../fixtures/fakeCdpBrowser.js"; -import { trackBrowser } from "./cdpBrowserProvider.support.js"; +import { frameNavigated, trackBrowser } from "./cdpBrowserProvider.support.js"; const observeDuringClockSampling = async ( commandEvents: NonNullable, ) => { - const browser = await startFakeCdpBrowser({ + return observeWithFake({ eventsBeforeReply: ["Performance.getMetrics"], commandResult: (command) => command.method === "Performance.getMetrics" @@ -19,6 +19,10 @@ const observeDuringClockSampling = async ( : undefined, commandEvents, }); +}; + +const observeWithFake = async (options: FakeOptions) => { + const browser = await startFakeCdpBrowser(options); trackBrowser(browser); const result = await new CdpBrowserProvider().observeSession( observeWebSessionInputSchema.parse({ @@ -32,34 +36,47 @@ const observeDuringClockSampling = async ( return { browser, observation: result.value }; }; +const laterInWindowRequest = ( + command: Parameters>[0], + origin: string, +) => ({ + method: "Network.requestWillBeSent", + sessionId: command.sessionId, + params: { + requestId: "later-in-window-request", + timestamp: 21, + type: "Document", + frameId: "frame-main", + redirectResponse: { url: `${origin}/prior` }, + request: { url: `${origin}/later` }, + }, +}); + +const expectNoCommittedFrameWithInvalidTimeline = ( + observation: Awaited>["observation"], +) => { + expect(observation.timeline).not.toContainEqual( + expect.objectContaining({ type: "navigation_committed" }), + ); + expect(observation.completeness.excluded).toContainEqual( + expect.objectContaining({ + section: "timeline", + reason: "invalid_protocol_value", + }), + ); +}; + it("fails closed on an untimestamped cross-origin main-frame commit during clock sampling", async () => { const { observation } = await observeDuringClockSampling( (command, origin) => { if (command.method !== "Performance.getMetrics") return undefined; return [ - { - method: "Page.frameNavigated", - sessionId: command.sessionId, - params: { - frame: { - id: "frame-main", - loaderId: "outside-loader", - url: "https://outside.invalid/path", - }, - }, - }, - { - method: "Network.requestWillBeSent", - sessionId: command.sessionId, - params: { - requestId: "later-in-window-request", - timestamp: 21, - type: "Document", - frameId: "frame-main", - redirectResponse: { url: `${origin}/prior` }, - request: { url: `${origin}/later` }, - }, - }, + frameNavigated(command, { + id: "frame-main", + loaderId: "outside-loader", + url: "https://outside.invalid/path", + }), + laterInWindowRequest(command, origin), ]; }, ); @@ -80,6 +97,93 @@ it("fails closed on an untimestamped cross-origin main-frame commit during clock ); }); +it("keeps an unidentifiable frame commit unknown without inventing a main-frame event", async () => { + const { observation } = await observeDuringClockSampling((command) => + command.method === "Performance.getMetrics" + ? [frameNavigated(command, { url: "https://outside.invalid/unknown" })] + : undefined, + ); + + expect(observation.window.end_reason).toBe("window_elapsed"); + expectNoCommittedFrameWithInvalidTimeline(observation); +}); + +it("marks a frame event with absent parameters as unknown timeline evidence", async () => { + const { observation } = await observeDuringClockSampling((command) => + command.method === "Performance.getMetrics" + ? [ + { + method: "Page.frameNavigated", + sessionId: command.sessionId, + params: undefined, + }, + ] + : undefined, + ); + + expectNoCommittedFrameWithInvalidTimeline(observation); + expect(observation.completeness.unavailable_sections).toContain("timeline"); +}); + +it("records a malformed child identity without treating it as the main frame", async () => { + const { observation } = await observeDuringClockSampling((command) => + command.method === "Performance.getMetrics" + ? [ + frameNavigated(command, { + id: 3, + parentId: "frame-main", + url: "https://outside.invalid/child", + }), + ] + : undefined, + ); + + expect(observation.window.end_reason).toBe("window_elapsed"); + expectNoCommittedFrameWithInvalidTimeline(observation); +}); + +it("updates the main frame from a pre-arm root replacement without inventing a navigation row", async () => { + const { observation } = await observeDuringClockSampling((command, origin) => + command.method === "Performance.getMetrics" + ? [ + frameNavigated(command, { + id: "replacement-main", + loaderId: "replacement-loader", + url: `${origin}/app?token=frame-secret`, + }), + { + method: "Page.lifecycleEvent", + sessionId: command.sessionId, + params: { + frameId: "replacement-main", + loaderId: "replacement-loader", + name: "DOMContentLoaded", + timestamp: 21, + }, + }, + ] + : undefined, + ); + + expect(observation.timeline).not.toContainEqual( + expect.objectContaining({ type: "navigation_committed" }), + ); + expect(observation.timeline).toContainEqual( + expect.objectContaining({ + type: "lifecycle", + frame_id: "replacement-main", + loader_id: "replacement-loader", + timestamp: 21, + detail: "domcontentloaded", + }), + ); + expect(observation.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("Buffered events with unknown timing"), + ]), + ); +}); + it("fails closed on an untimestamped out-of-scope main-frame request during clock sampling", async () => { const { observation } = await observeDuringClockSampling( (command, origin) => { @@ -94,18 +198,7 @@ it("fails closed on an untimestamped out-of-scope main-frame request during cloc reason: "scriptInitiated", }, }, - { - method: "Network.requestWillBeSent", - sessionId: command.sessionId, - params: { - requestId: "later-in-window-request", - timestamp: 21, - type: "Document", - frameId: "frame-main", - redirectResponse: { url: `${origin}/prior` }, - request: { url: `${origin}/later` }, - }, - }, + laterInWindowRequest(command, origin), ]; }, ); @@ -191,7 +284,7 @@ it.each([ ] as const)( "ends on buffered untimestamped %s before collecting later events", async (_method, terminalEvent) => { - const browser = await startFakeCdpBrowser({ + const { observation } = await observeWithFake({ eventsBeforeReply: ["Performance.getMetrics"], commandResult: (command) => command.method === "Performance.getMetrics" @@ -216,35 +309,25 @@ it.each([ ]; }, }); - trackBrowser(browser); - const result = await new CdpBrowserProvider().observeSession( - observeWebSessionInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 20, - }), - ); - if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("target_terminated"); - expect(result.value.timeline).toContainEqual( + expect(observation.window.end_reason).toBe("target_terminated"); + expect(observation.timeline).toContainEqual( expect.objectContaining({ type: "target_terminated", timestamp: null, }), ); - expect(result.value.timeline).not.toContainEqual( + expect(observation.timeline).not.toContainEqual( expect.objectContaining({ request_id: "after-terminal-event" }), ); - expect(result.value.limitations).toContain( + expect(observation.limitations).toContain( "Timeline rows without a comparable producer timestamp have a null timestamp. Buffered events with unknown timing are retained as context only; they do not establish activity inside the selected observation window.", ); }, ); it("represents absent timeline timestamps as unknown and preserves an observed zero", async () => { - const browser = await startFakeCdpBrowser({ + const { browser, observation } = await observeWithFake({ eventsBeforeReply: ["Page.setLifecycleEventsEnabled"], commandEvents: (command, origin) => { if (command.method !== "Page.setLifecycleEventsEnabled") return undefined; @@ -271,18 +354,8 @@ it("represents absent timeline timestamps as unknown and preserves an observed z ]; }, }); - trackBrowser(browser); - const result = await new CdpBrowserProvider().observeSession( - observeWebSessionInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 20, - }), - ); - if (!result.ok) throw result.error; - expect(result.value.timeline).toEqual( + expect(observation.timeline).toEqual( expect.arrayContaining([ expect.objectContaining({ type: "same_document_navigation", diff --git a/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts b/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts index b768485989..6657769292 100644 --- a/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts +++ b/tests/boundary/browser/cdpScreenshotDocumentIdentity.test.ts @@ -1,26 +1,33 @@ import { describe, expect, it, onTestFinished } from "vitest"; import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; import { captureWebScreenshotInputSchema } from "../../../src/domain/webScreenshot.js"; -import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; +import { + startFakeCdpBrowser, + type FakeOptions, +} from "../../fixtures/fakeCdpBrowser.js"; +import { frameNavigated } from "./cdpBrowserProvider.support.js"; + +const captureWithFake = async (options: FakeOptions) => { + const browser = await startFakeCdpBrowser(options); + onTestFinished(() => browser.close()); + const result = await new CdpBrowserProvider().captureScreenshot( + captureWebScreenshotInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + }), + ); + return { browser, result }; +}; describe("screenshot committed document identity", () => { it.each([ ["missing screenshot data", {}], ["invalid base64 screenshot data", { data: "not base64%%%" }], ])("reports %s as a protocol error", async (_label, result) => { - const browser = await startFakeCdpBrowser({ + const { result: observation } = await captureWithFake({ commandResult: ({ method }) => method === "Page.captureScreenshot" ? result : undefined, }); - onTestFinished(async () => { - await browser.close(); - }); - const observation = await new CdpBrowserProvider().captureScreenshot( - captureWebScreenshotInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - }), - ); expect(observation).toMatchObject({ ok: false, error: { reason: "protocol_error" }, @@ -37,22 +44,12 @@ describe("screenshot committed document identity", () => { "base64", ); bytes.writeUInt32BE(value, offset); - const browser = await startFakeCdpBrowser({ + const { result } = await captureWithFake({ commandResult: ({ method }) => method === "Page.captureScreenshot" ? { data: bytes.toString("base64") } : undefined, }); - onTestFinished(async () => { - await browser.close(); - }); - const result = await new CdpBrowserProvider().captureScreenshot( - captureWebScreenshotInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - }), - ); expect(result).toMatchObject({ ok: false, error: { reason: "protocol_error" }, @@ -61,17 +58,7 @@ describe("screenshot committed document identity", () => { it.each(["loader-main", "loader-reloaded"])( "distinguishes %s for the same URL", async (screenshotDocumentLoader) => { - const browser = await startFakeCdpBrowser({ screenshotDocumentLoader }); - onTestFinished(async () => { - await browser.close(); - }); - const result = await new CdpBrowserProvider().captureScreenshot( - captureWebScreenshotInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - }), - ); + const { result } = await captureWithFake({ screenshotDocumentLoader }); if (screenshotDocumentLoader === "loader-main") expect(result.ok).toBe(true); else @@ -81,4 +68,159 @@ describe("screenshot committed document identity", () => { }); }, ); + + it("defers an unidentifiable navigation failure until screenshot cleanup completes", async () => { + const { browser, result } = await captureWithFake({ + commandEvents: (command) => + command.method === "Page.captureScreenshot" + ? [ + { + method: "Page.frameNavigated", + ...(command.sessionId === undefined + ? {} + : { sessionId: command.sessionId }), + params: undefined, + }, + ] + : undefined, + }); + expect(result).toMatchObject({ + ok: false, + error: { reason: "protocol_error" }, + }); + await expect.poll(() => browser.activeConnectionCount()).toBe(0); + }); +}); + +describe("screenshot frame identity transitions", () => { + it("rejects a self-parented root candidate as malformed identity", async () => { + const { browser, result } = await captureWithFake({ + commandEvents: (command, origin) => + command.method === "Page.captureScreenshot" + ? [ + frameNavigated(command, { + id: "candidate-root", + parentId: "candidate-root", + url: `${origin}/app?token=frame-secret`, + }), + ] + : undefined, + }); + expect(result).toMatchObject({ + ok: false, + error: { reason: "protocol_error" }, + }); + await expect.poll(() => browser.activeConnectionCount()).toBe(0); + }); + + it("keeps a malformed known child as screenshot coverage", async () => { + const { result } = await captureWithFake({ + commandEvents: (command) => + command.method === "Page.captureScreenshot" + ? [ + frameNavigated(command, { + id: 7, + parentId: "frame-main", + url: "https://outside.invalid/child", + }), + ] + : undefined, + }); + if (!result.ok) throw result.error; + expect(result.value.completeness.excluded).toContainEqual( + expect.objectContaining({ + section: "screenshot", + reason: "invalid_protocol_value", + }), + ); + }); + + it("detects a same-URL root replacement and scopes later events to its new identity", async () => { + const { result } = await captureWithFake({ + commandEvents: (command, origin) => + command.method === "Page.captureScreenshot" + ? [ + frameNavigated(command, { + id: "replacement-main", + loaderId: "loader-main", + url: `${origin}/app?token=frame-secret`, + }), + { + method: "Page.navigatedWithinDocument", + ...(command.sessionId === undefined + ? {} + : { sessionId: command.sessionId }), + params: { + frameId: "replacement-main", + url: "https://outside.invalid/path", + }, + }, + ] + : undefined, + }); + expect(result).toMatchObject({ + ok: false, + error: { reason: "target_not_allowed" }, + }); + }); +}); + +describe("screenshot navigation event applicability", () => { + it("ignores non-navigation events with selected-frame or malformed parameters", async () => { + const { result } = await captureWithFake({ + commandEvents: (command) => + command.method === "Page.captureScreenshot" + ? [ + { + method: "Runtime.consoleAPICalled", + sessionId: command.sessionId, + params: { + type: "log", + args: [], + executionContextId: 1, + timestamp: 1, + }, + }, + { + method: "Page.lifecycleEvent", + sessionId: command.sessionId, + params: { + frameId: "frame-main", + loaderId: "loader-main", + name: "DOMContentLoaded", + timestamp: 1, + }, + }, + { + method: "Runtime.consoleAPICalled", + sessionId: command.sessionId, + params: { frameId: "frame-main", args: "not-an-array" }, + }, + ] + : undefined, + }); + expect(result.ok).toBe(true); + }); + + it.each([ + "Page.frameRequestedNavigation", + "Page.navigatedWithinDocument", + ] as const)("ignores a known non-main frame's %s event", async (method) => { + const { result } = await captureWithFake({ + commandEvents: (command) => + command.method === "Page.captureScreenshot" + ? [ + { + method, + sessionId: command.sessionId, + params: { + frameId: "frame-child", + url: "https://outside.invalid/child", + }, + }, + ] + : undefined, + }); + expect(result.ok).toBe(true); + }); }); diff --git a/tests/boundary/browser/cdpWebRuntime.test.ts b/tests/boundary/browser/cdpWebRuntime.test.ts index 7b5c772ca4..db44df1868 100644 --- a/tests/boundary/browser/cdpWebRuntime.test.ts +++ b/tests/boundary/browser/cdpWebRuntime.test.ts @@ -109,6 +109,45 @@ describe("native runtime producer boundary", () => { }); describe("runtime window and producer granularity", () => { + it("fails malformed frame identity and completes execution cleanup", async () => { + const { browser, provider, execution } = await fixture(); + const result = await provider.observeExecution( + { ...execution, observation_ms: 30_000 }, + { + progress: { + report: async (update) => { + if (update.completed !== 1) return; + browser.emitEvent({ + method: "Page.frameNavigated", + sessionId: "session-1", + params: { + frame: { + id: "other-frame", + parentId: 7, + url: "https://example.test/child", + }, + }, + }); + }, + }, + }, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "protocol_error" }, + }); + expect( + browser.commands.some( + (command) => command.method === "Profiler.stopPreciseCoverage", + ), + ).toBe(true); + expect( + browser.commands.some( + (command) => command.method === "Target.detachFromTarget", + ), + ).toBe(true); + }); + it("ends at document replacement and leaves coverage explicitly unavailable", async () => { const { browser, provider, execution } = await fixture(); const result = await provider.observeExecution( From 84c89d1107cda017509bc3cd851573d073f9906d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:26:20 +0800 Subject: [PATCH 31/35] test(javascript): separate heap pressure from graph admission proof Match memory failures to the exact source identity instead of graph ordering. Preserve CLI parse, file-evidence, partial-result, later-source and unchanged-input checks across valid memory-pressure outcomes. Use a controlled parent heap envelope with a real owned analyzer to prove module retention and graph rejection independently of host GC timing. Validation: full check:ci passed; 17 focused tests passed, with the final 4 changed tests rerun successfully. Independent test-strength and isolation review passed. The earlier hosted failure exposed missing graph-only fields; its exact alternate memory stage was not captured. --- .../javascriptCheckpointReserveCli.test.ts | 92 +++++++++++----- .../javascriptParentResultAdmission.test.ts | 100 ++++++++++++++++++ 2 files changed, 166 insertions(+), 26 deletions(-) create mode 100644 tests/boundary/javascript/javascriptParentResultAdmission.test.ts diff --git a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts index 32d113781e..5e612caf9e 100644 --- a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts +++ b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts @@ -9,11 +9,54 @@ import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/j import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +const assertRetainedMemoryFailure = (value: unknown, path: string): void => { + const parsed = z + .object({ + code: z.literal("resource_constraint"), + message: z.string(), + details: z.object({ + operation: z.literal("analyze_javascript_application"), + resource: z.literal("memory"), + reported_limits: z.record(z.string(), z.unknown()), + }), + remediation: z.object({ action: z.string() }), + }) + .safeParse(value); + if (!parsed.success) + throw new Error( + `Expected a typed memory constraint for ${path}; got ${JSON.stringify(value)}`, + { cause: parsed.error }, + ); + const failure = parsed.data; + const limits = failure.details.reported_limits; + const graphAdmission = z + .object({ + parent_available_result_budget_bytes: z.number(), + module_retention_estimate_bytes: z.number(), + graph_retention_estimate_bytes: z.number(), + }) + .safeParse(limits); + if (graphAdmission.success) { + expect( + graphAdmission.data.module_retention_estimate_bytes + + graphAdmission.data.graph_retention_estimate_bytes, + ).toBeGreaterThan(graphAdmission.data.parent_available_result_budget_bytes); + expect(failure.message).toContain( + `Semantic graph for ${path} exceeds the parent's remaining result budget`, + ); + expect(failure.remediation.action).toContain("parent result budget"); + expect(failure.remediation.action).toContain( + "retained static and module facts", + ); + return; + } +}; + cliTest.skipIf(process.versions.bun !== undefined).for([ { laterSources: 1, heapMb: 640, methodsCount: 400, retainGraph: true }, { laterSources: 10, heapMb: 640, methodsCount: 400, retainGraph: true }, - // Even an otherwise empty parent cannot admit this graph after reserving - // transient decoding and the next checkpoint, independent of GC timing. + // Exercise retained evidence and later checkpoints under real heap pressure. + // The controlled worker test proves graph-only admission independently of GC. { laterSources: 1, heapMb: 448, methodsCount: 600, retainGraph: false }, ])( "preserves evidence with $laterSources later checkpoints and a $heapMb MiB parent heap", @@ -50,11 +93,18 @@ cliTest.skipIf(process.versions.bun !== undefined).for([ ); expect(result.statistics.parsed_javascript_files).toBe(laterSources + 1); expect(result.statistics.parse_failures).toBe(0); - const failures = result.graph.nodes.flatMap(({ observations }) => - observations.flatMap(({ properties }) => + const failures = result.graph.nodes.flatMap((node) => + node.observations.flatMap(({ properties }) => properties.analysis_failure === undefined ? [] - : [properties.analysis_failure], + : [ + { + kind: node.kind, + identity: node.identity, + path: properties.path, + failure: properties.analysis_failure, + }, + ], ), ); const methods = result.semantic_graph.nodes.filter( @@ -103,28 +153,18 @@ cliTest.skipIf(process.versions.bun !== undefined).for([ if (retainGraph) expect(failures).toEqual([]); expect(await readFile(join(root, name), "utf8")).toBe(text); if (!retainGraph && name === "a-large.mjs") { - const failure = z - .object({ - code: z.literal("resource_constraint"), - remediation: z.object({ action: z.string() }), - details: z.object({ - reported_limits: z.object({ - parent_available_result_budget_bytes: z.number(), - module_retention_estimate_bytes: z.number(), - graph_retention_estimate_bytes: z.number(), - }), - }), - }) - .parse(failures[0]); - const limits = failure.details.reported_limits; - expect( - limits.module_retention_estimate_bytes + - limits.graph_retention_estimate_bytes, - ).toBeGreaterThan(limits.parent_available_result_budget_bytes); - expect(failure.remediation.action).toContain("parent result budget"); - expect(failure.remediation.action).toContain( - "retained static and module facts", + const failureRecord = failures.find( + ({ kind, identity, path }) => + kind === "unknown" && + path === name && + identity.strategy === "artifact-local-key" && + identity.artifact_sha256 === digest && + identity.namespace === "static-parse-scope" && + identity.key === `analyze-javascript-source:${name}`, ); + if (failureRecord === undefined) + throw new Error(`Missing analysis failure for ${name}`); + assertRetainedMemoryFailure(failureRecord.failure, name); } } expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu); diff --git a/tests/boundary/javascript/javascriptParentResultAdmission.test.ts b/tests/boundary/javascript/javascriptParentResultAdmission.test.ts new file mode 100644 index 0000000000..d1e1f7a017 --- /dev/null +++ b/tests/boundary/javascript/javascriptParentResultAdmission.test.ts @@ -0,0 +1,100 @@ +import { createHash } from "node:crypto"; +import { afterAll, expect, it, vi } from "vitest"; + +// Give the parent a stable 64 MiB result envelope after its transfer reserve. +// The child analyzer remains a real separately owned Node process with its +// normal heap and real transfer files/descriptors. +vi.mock("node:v8", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getHeapStatistics: () => ({ + ...actual.getHeapStatistics(), + heap_size_limit: process.memoryUsage().heapUsed + 320 * 1024 * 1024, + }), + }; +}); + +import { JavaScriptAnalysisWorker } from "../../../src/javascript/analysis/JavaScriptAnalysisWorker.js"; +import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; + +afterAll(() => { + vi.doUnmock("node:v8"); + vi.resetModules(); +}); + +it("retains module facts and declines only the graph when its real transfer exceeds parent admission", async () => { + const methodsCount = 600; + const text = Array.from( + { length: methodsCount }, + (_, index) => + `export function method_${String(index)}(input) { const value = input + ${String(index)}; const pair = { value }; return pair.value; }`, + ).join("\n"); + const file: JavaScriptArtifactFile = { + path: "a-large.mjs", + sha256: createHash("sha256").update(text).digest("hex"), + container_sha256: "1".repeat(64), + bytes: Buffer.byteLength(text), + inventory_artifact_id: "parent-admission-fixture", + kind: "javascript", + unpacked: false, + text: { included: true, value: text }, + }; + const worker = new JavaScriptAnalysisWorker({ + heapMb: 1024, + timeoutMs: 30_000, + }); + try { + const result = await worker.analyze(file, { + nodes: 20_000, + unknowns: 20_000, + }); + if (result.ok) + throw new Error( + "Expected the parent to decline the large semantic graph", + ); + const { error, javascript, module, projection } = result.error; + expect(javascript?.parse_status).toBe("complete"); + expect(module?.callables).toHaveLength(methodsCount); + expect(projection).toBeNull(); + expect(error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "memory", + reason: + "Semantic graph for a-large.mjs exceeds the parent's remaining result budget; static and module facts were retained", + }); + if (!(error instanceof AnalysisResourceConstraintError)) + throw new Error("Expected a typed resource constraint"); + expect(error.reportedLimits).toMatchObject( + process.versions.bun === undefined + ? { worker_old_space_mb: 1024 } + : { worker_heap_limit_bytes: null }, + ); + const limits = error.reportedLimits; + if (limits === null) + throw new Error("Parent admission omitted its measured limits"); + expect(limits.parent_result_budget_bytes).toBeLessThanOrEqual( + 64 * 1024 * 1024, + ); + expect(limits.parent_result_budget_bytes).toBeGreaterThan(0); + const moduleEstimate = limits.module_retention_estimate_bytes; + const graphEstimate = limits.graph_retention_estimate_bytes; + const available = limits.parent_available_result_budget_bytes; + expect(typeof moduleEstimate).toBe("number"); + expect(typeof graphEstimate).toBe("number"); + expect(typeof available).toBe("number"); + if ( + typeof moduleEstimate !== "number" || + typeof graphEstimate !== "number" || + typeof available !== "number" + ) + throw new Error( + "Parent admission omitted its measured transfer estimates", + ); + expect(moduleEstimate).toBeLessThanOrEqual(available); + expect(moduleEstimate + graphEstimate).toBeGreaterThan(available); + } finally { + await worker.close(); + } +}); From 1502cfbc8ca0d1d1d86eebf334018dfb418bbac0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:37:15 +0800 Subject: [PATCH 32/35] fix(apple): resolve absolute dylib paths inside the selected root Preserve load-command paths as evidence while passing a separate root-relative lookup path to the bounded tree resolver. Apply the same containment rule to absolute install names and rpath bases, including caller-selected root symlink spellings. Validation: 140 source tests, full check:ci, docs and runtime build passed. Compiler-built literal, rpath and relative fixtures passed CLI/stdio MCP parity, preserved both dependency edges and left all bytes unchanged. External system paths stayed outside-target; relative control was unchanged. Fixtures were not executed, and actual dyld execution was not qualified. Closes #5637 --- docs/apple-application-analysis.md | 10 +- src/artifacts/apple/DylibResolutionReader.ts | 24 +- .../apple/MachoLoadCommandReader.test.ts | 237 +++++++++++++++++- src/domain/apple/dyldPaths.ts | 16 +- src/domain/apple/dylibResolution.test.ts | 1 + src/domain/apple/dylibResolution.ts | 43 +++- 6 files changed, 312 insertions(+), 19 deletions(-) diff --git a/docs/apple-application-analysis.md b/docs/apple-application-analysis.md index 8425877e62..926aae0d41 100644 --- a/docs/apple-application-analysis.md +++ b/docs/apple-application-analysis.md @@ -228,9 +228,13 @@ root supplies. Every edge below it has `loader_conditional: true`, and a request that reuses it by install name is `conditional` too. Findings on those edges say that they apply only if the conditional image loads. -Absolute install names such as `/usr/lib/libSystem.B.dylib` are -`outside-target`. On macOS 11 and later most of them live in the dyld shared -cache rather than on disk, so REA does not check them against the host. +Absolute install names that name a path beneath the analyzed root are checked +within that root, with symlinks and `..` resolved by the same bounded tree +resolver. Absolute paths outside the root, such as +`/usr/lib/libSystem.B.dylib`, are `outside-target`; on macOS 11 and later most +system libraries live in the dyld shared cache rather than on disk, so REA does +not check them against the host. Mach-O path strings are not converted from +`file:` URLs or looked up through unrelated host paths. `findings` are derived, not observed: diff --git a/src/artifacts/apple/DylibResolutionReader.ts b/src/artifacts/apple/DylibResolutionReader.ts index 42219ddb20..30e1205275 100644 --- a/src/artifacts/apple/DylibResolutionReader.ts +++ b/src/artifacts/apple/DylibResolutionReader.ts @@ -1,6 +1,6 @@ import type { BigIntStats } from "node:fs"; import { lstat, readlink, realpath } from "node:fs/promises"; -import { join, relative, sep } from "node:path"; +import { isAbsolute, join, relative, sep } from "node:path"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; import { resolveTreePath } from "../../domain/apple/dyldPaths.js"; @@ -36,10 +36,25 @@ class FilesystemTreeView implements DylibTreeView { constructor( private readonly root: string, + private readonly sourceRoot: string, private readonly files: DylibFileAccess, private readonly signal?: AbortSignal, ) {} + rootRelativePath(absolutePath: string): string | undefined { + const roots = isAbsolute(this.sourceRoot) + ? [this.root, this.sourceRoot] + : [this.root]; + for (let root of roots) { + while (root.length > 1 && root.endsWith(sep)) root = root.slice(0, -1); + if (absolutePath === root) return ""; + const prefix = root === sep ? sep : `${root}${sep}`; + if (absolutePath.startsWith(prefix)) + return absolutePath.slice(prefix.length).split(sep).join("/"); + } + return undefined; + } + entry(path: string): Promise { const cached = this.#entries.get(path); if (cached !== undefined) return cached; @@ -113,7 +128,12 @@ export const traceDylibResolution = async (options: { .split(sep) .join("/"); assertArtifactReadActive(options.signal); - const view = new FilesystemTreeView(root, files, options.signal); + const view = new FilesystemTreeView( + root, + options.rootPath, + files, + options.signal, + ); const { roots: requested, unclassified } = parsed.data.roots !== undefined ? { roots: parsed.data.roots, unclassified: [] } diff --git a/src/artifacts/apple/MachoLoadCommandReader.test.ts b/src/artifacts/apple/MachoLoadCommandReader.test.ts index fdd9a79815..94dd4891c1 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.test.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.test.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdir, rename, writeFile } from "node:fs/promises"; +import { mkdir, realpath, rename, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -142,6 +142,241 @@ it("uses legacy simulator and device platform facts when resolving dylibs", asyn ); }); +describe("absolute dyld candidate paths at the filesystem reader boundary", () => { + it.each(["install-name", "rpath"] as const)( + "resolves an absolute in-root %s while preserving its source path", + async (kind) => { + const root = await createTestTempDirectory("rea-dylib-absolute-in-root-"); + const canonicalRoot = await realpath(root); + const bridgePath = "Frameworks/libBridge.dylib"; + const leafPath = "Frameworks/libLeaf.dylib"; + const bridgeAbsolute = `${canonicalRoot}/${bridgePath}`; + const leafName = "@loader_path/libLeaf.dylib"; + const executable = machoImage({ + commands: [ + buildVersionCommand(1), + ...(kind === "rpath" + ? [rpathCommand(`${canonicalRoot}/Frameworks`)] + : []), + dylibCommand( + LC.LOAD_DYLIB, + kind === "rpath" ? "@rpath/libBridge.dylib" : bridgeAbsolute, + ), + ], + }); + const bridge = machoImage({ + fileType: FILE_TYPE.dylib, + commands: [ + buildVersionCommand(1), + dylibCommand(LC.LOAD_DYLIB, leafName), + ], + }); + const leaf = machoImage({ + fileType: FILE_TYPE.dylib, + commands: [buildVersionCommand(1)], + }); + await mkdir(join(root, "Frameworks"), { recursive: true }); + await writeFile(join(root, "App"), executable); + await writeFile(bridgeAbsolute, bridge); + await writeFile(join(root, leafPath), leaf); + + const trace = await traceDylibResolution({ + rootPath: root, + targetPath: join(root, "App"), + targetSha256: createHash("sha256").update(executable).digest("hex"), + enumerateRoots: false, + parameters: { roots: ["App"] }, + }); + + expect(trace.edges[0]).toMatchObject({ + install_name: + kind === "rpath" ? "@rpath/libBridge.dylib" : bridgeAbsolute, + candidates: [ + { + path: bridgeAbsolute, + ...(kind === "rpath" + ? { rpath: `${canonicalRoot}/Frameworks` } + : {}), + outcome: "resolved", + resolved_path: bridgePath, + }, + ], + resolution: { status: "resolved", image: bridgePath }, + }); + expect(trace.edges).toContainEqual( + expect.objectContaining({ + loader: bridgePath, + install_name: leafName, + resolution: { status: "resolved", image: leafPath }, + }), + ); + expect(trace.images.map(({ path }) => path)).toEqual([ + "App", + bridgePath, + leafPath, + ]); + }, + ); + + it("keeps a genuinely external absolute candidate outside the analyzed root", async () => { + const root = await createTestTempDirectory("rea-dylib-absolute-outside-"); + const canonicalRoot = await realpath(root); + const executable = machoImage({ + commands: [ + buildVersionCommand(1), + dylibCommand(LC.LOAD_DYLIB, "/usr/lib/libSystem.B.dylib"), + dylibCommand(LC.LOAD_DYLIB, `${canonicalRoot}/../outside.dylib`), + dylibCommand( + LC.LOAD_DYLIB, + `file://${canonicalRoot}/Frameworks/libBridge.dylib`, + ), + ], + }); + await writeFile(join(root, "App"), executable); + const trace = await traceDylibResolution({ + rootPath: root, + targetPath: join(root, "App"), + targetSha256: createHash("sha256").update(executable).digest("hex"), + enumerateRoots: false, + parameters: { roots: ["App"] }, + }); + + expect(trace.edges[0]).toMatchObject({ + candidates: [ + { + path: "/usr/lib/libSystem.B.dylib", + outcome: "outside-target", + resolved_path: null, + }, + ], + resolution: { status: "undetermined", image: null }, + }); + expect(trace.edges[1]).toMatchObject({ + candidates: [{ outcome: "escapes-target", resolved_path: null }], + resolution: { status: "undetermined", image: null }, + }); + expect(trace.edges[2]).toMatchObject({ + install_name: `file://${canonicalRoot}/Frameworks/libBridge.dylib`, + candidates: [ + { + path: `file://${canonicalRoot}/Frameworks/libBridge.dylib`, + outcome: "undetermined", + resolved_path: null, + }, + ], + resolution: { status: "undetermined", image: null }, + }); + expect(trace.images.map(({ path }) => path)).toEqual(["App"]); + }); +}); + +describe("absolute dyld candidate traversal", () => { + it("maps an absolute candidate spelled through the selected root symlink", async () => { + const root = await createTestTempDirectory( + "rea-dylib-absolute-root-alias-", + ); + const aliasRoot = join(root, "bundle-alias"); + const bridgePath = "Frameworks/libBridge.dylib"; + const bridgeAbsolute = `${aliasRoot}/${bridgePath}`; + const executable = machoImage({ + commands: [ + buildVersionCommand(1), + dylibCommand(LC.LOAD_DYLIB, bridgeAbsolute), + ], + }); + const bridge = machoImage({ + fileType: FILE_TYPE.dylib, + commands: [buildVersionCommand(1)], + }); + await symlink(".", aliasRoot); + await mkdir(join(root, "Frameworks"), { recursive: true }); + await writeFile(join(root, "App"), executable); + await writeFile(join(root, bridgePath), bridge); + + const trace = await traceDylibResolution({ + rootPath: aliasRoot, + targetPath: `${aliasRoot}/App`, + targetSha256: createHash("sha256").update(executable).digest("hex"), + enumerateRoots: false, + parameters: { roots: ["App"] }, + }); + + expect(trace.edges[0]).toMatchObject({ + install_name: bridgeAbsolute, + candidates: [ + { + path: bridgeAbsolute, + outcome: "resolved", + resolved_path: bridgePath, + }, + ], + resolution: { status: "resolved", image: bridgePath }, + }); + }); + + it.skipIf(process.platform === "win32")( + "preserves symlink-then-parent traversal for an absolute in-root path", + async () => { + const root = await createTestTempDirectory("rea-dylib-absolute-symlink-"); + const canonicalRoot = await realpath(root); + const bridgePath = "Frameworks/Versions/libBridge.dylib"; + const decoyPath = "Frameworks/libBridge.dylib"; + const sourcePath = `${canonicalRoot}/Frameworks/Current/../libBridge.dylib`; + const executable = machoImage({ + commands: [ + buildVersionCommand(1), + dylibCommand(LC.LOAD_DYLIB, sourcePath), + ], + }); + const bridge = machoImage({ + fileType: FILE_TYPE.dylib, + commands: [ + buildVersionCommand(1), + dylibCommand(LC.LOAD_DYLIB, "@loader_path/libLeaf.dylib"), + ], + }); + const leaf = machoImage({ + fileType: FILE_TYPE.dylib, + commands: [buildVersionCommand(1)], + }); + await mkdir(join(root, "Frameworks/Versions/A"), { recursive: true }); + await symlink("Versions/A", join(root, "Frameworks/Current")); + await writeFile(join(root, "App"), executable); + await writeFile(join(root, bridgePath), bridge); + await writeFile(join(root, decoyPath), new Uint8Array([0, 1, 2])); + await writeFile(join(root, "Frameworks/Versions/libLeaf.dylib"), leaf); + + const trace = await traceDylibResolution({ + rootPath: root, + targetPath: join(root, "App"), + targetSha256: createHash("sha256").update(executable).digest("hex"), + enumerateRoots: false, + parameters: { roots: ["App"] }, + }); + + expect(trace.edges[0]).toMatchObject({ + candidates: [ + { + path: sourcePath, + outcome: "resolved", + resolved_path: bridgePath, + }, + ], + resolution: { status: "resolved", image: bridgePath }, + }); + expect(trace.edges).toContainEqual( + expect.objectContaining({ + loader: bridgePath, + resolution: { + status: "resolved", + image: "Frameworks/Versions/libLeaf.dylib", + }, + }), + ); + }, + ); +}); + it("selects matching arm64e subtype slices when tracing a mixed FAT image", async () => { const root = await createTestTempDirectory("rea-macho-arm64e-subtypes-"); const executablePath = "Contents/MacOS/App"; diff --git a/src/domain/apple/dyldPaths.ts b/src/domain/apple/dyldPaths.ts index af2764d291..7feb8da936 100644 --- a/src/domain/apple/dyldPaths.ts +++ b/src/domain/apple/dyldPaths.ts @@ -68,7 +68,13 @@ export const joinPath = (directory: string, rest: string): string => directory === "" ? rest : rest === "" ? directory : `${directory}/${rest}`; export type Expansion = - | { readonly scope: "target"; readonly path: string } + | { + readonly scope: "target"; + /** Source-facing expansion retained in candidate Evidence. */ + readonly path: string; + /** Root-relative lookup identity when `path` is an absolute source path. */ + readonly lookupPath?: string; + } | { readonly scope: "outside"; readonly path: string } | { readonly scope: "undetermined"; readonly path: string }; @@ -78,6 +84,7 @@ export const expandPrefix = ( context: { readonly ownerDirectory: string; readonly executable: string | null; + readonly rootRelativePath: (absolutePath: string) => string | undefined; }, ): Expansion => { const variable = /^@(executable_path|loader_path)(?:\/(.*))?$/u.exec(raw); @@ -92,7 +99,12 @@ export const expandPrefix = ( path: joinPath(directoryOf(context.executable), rest), }; } - if (raw.startsWith("/")) return { scope: "outside", path: raw }; + if (raw.startsWith("/")) { + const lookupPath = context.rootRelativePath(raw); + return lookupPath === undefined + ? { scope: "outside", path: raw } + : { scope: "target", path: raw, lookupPath }; + } return { scope: "undetermined", path: raw }; }; diff --git a/src/domain/apple/dylibResolution.test.ts b/src/domain/apple/dylibResolution.test.ts index f1c44572b9..dd4c18df28 100644 --- a/src/domain/apple/dylibResolution.test.ts +++ b/src/domain/apple/dylibResolution.test.ts @@ -80,6 +80,7 @@ const memoryView = ( entries.set(path, { kind: "symlink", target }); } return { + rootRelativePath: () => undefined, entry: (path) => Promise.resolve(entries.get(path)), image: (path) => { const facts = files[path]; diff --git a/src/domain/apple/dylibResolution.ts b/src/domain/apple/dylibResolution.ts index 7e46976390..1adfdc79fc 100644 --- a/src/domain/apple/dylibResolution.ts +++ b/src/domain/apple/dylibResolution.ts @@ -114,6 +114,8 @@ export type DylibTreeEntry = /** Read-only, lazily probed view of the analyzed root. Paths are root-relative. */ export interface DylibTreeView { + /** Map an absolute source path only when it is spelled beneath this root. */ + rootRelativePath(absolutePath: string): string | undefined; entry(path: string): Promise; image(path: string): Promise; } @@ -252,6 +254,24 @@ interface CandidateTemplate { readonly rpathOwner: string | null; } +const expandCandidatePath = ( + raw: string, + ownerDirectory: string, + context: ProcessContext, +): Expansion => + expandPrefix(raw, { + ownerDirectory, + executable: context.executable, + rootRelativePath: (absolutePath) => + context.view.rootRelativePath(absolutePath), + }); + +const appendRpath = (base: string, rest: string): string => { + if (base === "") return rest; + if (base === "/") return `/${rest}`; + return `${base.replace(/\/+$/u, "")}/${rest}`; +}; + const candidateTemplates = ( dependency: MachoDependency, loader: string, @@ -266,10 +286,7 @@ const candidateTemplates = ( : "literal"; return [ { - expansion: expandPrefix(name, { - ownerDirectory: directoryOf(loader), - executable: context.executable, - }), + expansion: expandCandidatePath(name, directoryOf(loader), context), source, rpath: null, rpathOwner: null, @@ -280,16 +297,17 @@ const candidateTemplates = ( const owners = [...(context.loaded.get(loader)?.chain ?? [loader])].reverse(); return owners.flatMap((owner) => (context.loaded.get(owner)?.slice.rpaths ?? []).map((rpath) => { - const base = expandPrefix(rpath, { - ownerDirectory: directoryOf(owner), - executable: context.executable, - }); + const base = expandCandidatePath(rpath, directoryOf(owner), context); + const lookupPath = base.scope === "target" ? base.lookupPath : undefined; const expansion: Expansion = { scope: base.scope, path: - base.scope === "outside" - ? `${base.path.replace(/\/+$/u, "")}/${rest}` + base.scope === "outside" || lookupPath !== undefined + ? appendRpath(base.path, rest) : `${base.path}${base.path === "" ? "" : "/"}${rest}`, + ...(lookupPath === undefined + ? {} + : { lookupPath: appendRpath(lookupPath, rest) }), }; return { expansion, source: "rpath" as const, rpath, rpathOwner: owner }; }), @@ -310,7 +328,10 @@ const evaluateCandidate = async ( return { ...base, outcome: "outside-target", resolved_path: null }; if (template.expansion.scope === "undetermined") return { ...base, outcome: "undetermined", resolved_path: null }; - const lookup = await resolveTreePath(context.view, template.expansion.path); + const lookup = await resolveTreePath( + context.view, + template.expansion.lookupPath ?? template.expansion.path, + ); if (lookup.kind === "escapes") return { ...base, outcome: "escapes-target", resolved_path: null }; if (lookup.kind !== "file") From 9258df02cc414112d4fa91a425f0d599f43e2357 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:32:53 +0800 Subject: [PATCH 33/35] test(apple): compare asset catalogs from matching root workflows Compare explicit bundle-resource CLI and MCP results at the positive empty-inventory offset. Preserve a separate binary-bound MCP assertion with the exact fixture executable digest and complete metadata payload. Validation: reproduced the stale root-versus-binary parity failure on current main, then passed the macOS public CLI/MCP acceptance case across all five NIB fixtures. Focused lint, formatting and diff checks passed. No target code was executed. --- .../assetCatalogResourceKeys.test.ts | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/applications/assetCatalogResourceKeys.test.ts b/tests/acceptance/applications/assetCatalogResourceKeys.test.ts index 61d2105724..289ad3d253 100644 --- a/tests/acceptance/applications/assetCatalogResourceKeys.test.ts +++ b/tests/acceptance/applications/assetCatalogResourceKeys.test.ts @@ -97,6 +97,7 @@ cliTest.skipIf(process.platform !== "darwin")( const result = appleAssetCatalogResultSchema.parse( parseEvidence(inspected.json).normalized_result, ); + const executableSha256 = createHash("sha256").update(mach).digest("hex"); expect(result.catalogs).toEqual([]); expect(result).toMatchObject({ total_records: 0, @@ -122,13 +123,35 @@ cliTest.skipIf(process.platform !== "darwin")( }, ], ); - const opened = await call("open_binary", { path: app }); + const opened = await call("open_bundle_resources", { path: app }); expect(opened.isError).not.toBe(true); const mcp = await call("inspect_asset_catalog", { offset: 1 }); expect(mcp.isError).not.toBe(true); expect(parseEvidence(mcp.structuredContent).normalized_result).toEqual( result, ); + + const binaryOpened = await call("open_binary", { path: app }); + expect(binaryOpened.isError).not.toBe(true); + const binary = await call("inspect_asset_catalog", { offset: 1 }); + expect(binary.isError).not.toBe(true); + const binaryResult = appleAssetCatalogResultSchema.parse( + parseEvidence(binary.structuredContent).normalized_result, + ); + expect(binaryResult.target_sha256).toBe(executableSha256); + expect(binaryResult.resource_root).toBeUndefined(); + expect(binaryResult).toEqual({ + target_sha256: executableSha256, + catalogs: result.catalogs, + total_records: result.total_records, + offset: result.offset, + limit: result.limit, + records: result.records, + resource_key_matches: result.resource_key_matches, + next_offset: result.next_offset, + truncated: result.truncated, + limitations: result.limitations, + }); } }, 30_000, From d0067ba34a39ea47dd7e11bb2e6169bc0938cc3f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:17:12 +0800 Subject: [PATCH 34/35] fix(native)!: preserve Swift pointer nullability and import bindings Resolve relative pointers using field-owned ABI encodings. Keep null, raw, rebased, imported and unknown targets distinct, and retain imported conformance rows with exact Mach-O binding provenance. Decode legacy rebase membership with source-derived memory and work bounds, preserving validated prefix facts when streams are incomplete. Validation: check:ci, docs:check and build passed; 158 source tests and 20 schema/acceptance tests passed. Ten source-built Apple dispatch variants and four issue-specific CLI/MCP workflows passed without executing targets. Paired cold/warm CLI/MCP cost observations retained locally; model tokens and maximum-sized input runtime were not measured. Closes #5683 Closes #5685 BREAKING CHANGE: Swift conformance type_name and protocol_name can be null when names cannot be resolved; consumers must use typed type_binding and protocol_binding evidence. MachoImportBinding replaces the former Objective-C-specific binding type and schema. --- docs/native-investigation.md | 13 +- scripts/verify/apple/dispatch.mjs | 4 +- src/domain/native/objcSwiftMetadata.ts | 38 +- src/domain/symbolAnalysis.ts | 15 +- src/native/AppleDispatchCoverage.ts | 29 +- src/native/AppleDispatchMetadata.test.ts | 261 +++++++++ src/native/AppleDispatchMetadata.ts | 13 +- src/native/AppleMachoFixups.test.ts | 370 +++++++++++- src/native/AppleMachoFixups.ts | 551 +++++++++++++++--- .../AppleObjcClassImportBindings.test.ts | 36 ++ src/native/AppleObjcClassImportBindings.ts | 10 +- src/native/AppleSwiftDispatchFacets.ts | 320 ++++++++-- src/native/AppleSwiftRelativePointer.test.ts | 274 +++++++++ src/native/AppleSwiftVtables.test.ts | 355 ++++++----- src/native/AppleSwiftVtables.ts | 51 +- 15 files changed, 2026 insertions(+), 314 deletions(-) create mode 100644 src/native/AppleSwiftRelativePointer.test.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index f17f5bbc2f..60105b42c1 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -432,7 +432,18 @@ complete observation. Pointers are decoded through the image's fixups: - `LC_DYLD_CHAINED_FIXUPS` rebases and binds, including the authenticated arm64e formats; - - legacy `LC_DYLD_INFO` bind opcodes. + - legacy `LC_DYLD_INFO` rebase and bind opcodes. + + A Swift conformance whose type or protocol reference is an external bind + keeps its name as `null` and carries the raw dyld identity in + `type_binding` or `protocol_binding`; that record and its facet coverage are + partial. Consumers of this output must handle nullable conformance names + and use the typed binding fields instead of substituting a symbol as a + display name. Indirect relative-pointer records also retain their binding + identity. A decoded rebase to virtual address zero remains an address; only + an unfixed zero in a nullable ABI field is treated as null. Incomplete or + conflicting fixup streams keep affected pointers unresolved; coverage reports + parsing and resource limits while retaining validated prefix facts. So an external superclass resolves from its `_OBJC_CLASS_$_` bind (for example `NSObject`). The `pointer_fixups` coverage facet names the mechanism. diff --git a/scripts/verify/apple/dispatch.mjs b/scripts/verify/apple/dispatch.mjs index cd222e0a50..316437266d 100644 --- a/scripts/verify/apple/dispatch.mjs +++ b/scripts/verify/apple/dispatch.mjs @@ -30,7 +30,7 @@ const LINK_MODES = [ { mode: "dyld-info", flags: ["-Wl,-no_fixup_chains"], - fixups: "LC_DYLD_INFO rebase opcodes were not decoded", + fixups: "LC_DYLD_INFO rebase opcodes", swift: true, }, { mode: "chained", flags: [], fixups: "chained fixups:", swift: true }, @@ -70,6 +70,7 @@ const inspect = async (path) => { }; const checkObjc = (metadata, mode) => { + assert.equal(fixupCoverage(metadata)?.status, "complete"); const fixture = metadata.objc_classes.find( ({ name, is_meta_class: meta }) => name === "ReaDispatchFixture" && !meta, ); @@ -159,6 +160,7 @@ const checkObjc = (metadata, mode) => { }; const checkSwift = (metadata, mode) => { + assert.equal(fixupCoverage(metadata)?.status, "complete"); if ( !metadata.swift_dispatch_slots.some( (slot) => diff --git a/src/domain/native/objcSwiftMetadata.ts b/src/domain/native/objcSwiftMetadata.ts index 546a801aca..fca04b09c7 100644 --- a/src/domain/native/objcSwiftMetadata.ts +++ b/src/domain/native/objcSwiftMetadata.ts @@ -199,10 +199,24 @@ export const objcCategorySchema = z.strictObject({ evidence: z.array(nativeMetadataEvidenceSchema).min(1), }); +/** Exact Mach-O import binding observed at a pointer slot. */ +export const machoImportBindingSchema = z.strictObject({ + requested_address: z.string().min(1), + symbol: z.string().min(1), + binding_source: z.enum(["chained", "bind", "weak", "lazy"]), + library_ordinal: z.number().int().min(-3).max(Number.MAX_SAFE_INTEGER), + library: z.string().nullable(), + weak: z.boolean(), + addend: z.string().regex(/^-?\d+$/u), +}); +export type MachoImportBinding = z.infer; + /** Swift protocol conformance and its associated witness-table location. */ export const swiftConformanceSchema = z.strictObject({ - type_name: z.string().min(1), - protocol_name: z.string().min(1), + type_name: z.string().min(1).nullable(), + type_binding: machoImportBindingSchema.nullable().optional(), + protocol_name: z.string().min(1).nullable(), + protocol_binding: machoImportBindingSchema.nullable().optional(), module: z.string().nullable(), witness_table: nativeMetadataLocationSchema, location: nativeMetadataLocationSchema, @@ -218,6 +232,7 @@ export const swiftDispatchSlotSchema = z.strictObject({ requirement: z.string().nullable(), implementation: z.string().nullable(), implementation_address: z.string().nullable(), + implementation_binding: machoImportBindingSchema.nullable().optional(), thunk_address: z.string().nullable(), location: nativeMetadataLocationSchema, decode: nativeMetadataDecodeSchema, @@ -231,6 +246,8 @@ export const relativePointerSchema = z.strictObject({ displacement: z.number().int(), indirectable: z.boolean(), indirect: z.boolean().nullable(), + encoding_tag: z.number().int().nonnegative().optional(), + binding: machoImportBindingSchema.optional(), decode: nativeMetadataDecodeSchema, evidence: z.array(nativeMetadataEvidenceSchema).min(1), }); @@ -253,17 +270,7 @@ export const nativeMetadataCoverageSchema = z.strictObject({ decoded: z.number().int().nonnegative(), }); -/** Exact Mach-O import binding observed at a caller-selected pointer slot. */ -export const objcClassImportBindingSchema = z.strictObject({ - requested_address: z.string().min(1), - symbol: z.string().min(1), - binding_source: z.enum(["chained", "bind", "weak", "lazy"]), - library_ordinal: z.number().int().min(-3).max(Number.MAX_SAFE_INTEGER), - library: z.string().nullable(), - weak: z.boolean(), - addend: z.string().regex(/^-?\d+$/u), -}); -export const objcClassImportIdentitySchema = objcClassImportBindingSchema.omit({ +export const objcClassImportIdentitySchema = machoImportBindingSchema.omit({ requested_address: true, }); export const objcClassImportBindingCoverageSchema = z.strictObject({ @@ -273,9 +280,6 @@ export const objcClassImportBindingCoverageSchema = z.strictObject({ examined: z.number().int().nonnegative(), decoded: z.number().int().nonnegative(), }); -export type ObjcClassImportBinding = z.infer< - typeof objcClassImportBindingSchema ->; export type ObjcClassImportBindingCoverage = z.infer< typeof objcClassImportBindingCoverageSchema >; @@ -348,7 +352,7 @@ export const objcSwiftMetadataSchema = z.strictObject({ swift_conformances: z.array(swiftConformanceSchema).default([]), swift_dispatch_slots: z.array(swiftDispatchSlotSchema).default([]), swift_symbols: z.array(swiftSymbolSchema).default([]), - objc_class_import_bindings: z.array(objcClassImportBindingSchema).optional(), + objc_class_import_bindings: z.array(machoImportBindingSchema).optional(), objc_class_import_binding_coverage: objcClassImportBindingCoverageSchema.optional(), target_selection: machoTargetSelectionSchema.optional(), diff --git a/src/domain/symbolAnalysis.ts b/src/domain/symbolAnalysis.ts index f23e1ab8e3..838ac62cc1 100644 --- a/src/domain/symbolAnalysis.ts +++ b/src/domain/symbolAnalysis.ts @@ -1,7 +1,7 @@ import type { JsonValue } from "./jsonValue.js"; import type { AddressedName } from "./hopperValues.js"; import type { - ObjcClassImportBinding, + MachoImportBinding, ObjcClassImportBindingCoverage, } from "./native/objcSwiftMetadata.js"; import { @@ -31,7 +31,7 @@ export interface ObjcClassDiscoveryCoverage { } export interface ObjcClassImportBindingScan { - readonly bindings: readonly ObjcClassImportBinding[]; + readonly bindings: readonly MachoImportBinding[]; readonly coverage: ObjcClassDiscoveryCoverage; } @@ -247,22 +247,19 @@ const uniqueByAddressAndName = ( }; interface ObjcClassOutputEntry extends AddressedName { - readonly binding_identity?: Omit< - ObjcClassImportBinding, - "requested_address" - > & { + readonly binding_identity?: Omit & { readonly slot_addresses: readonly string[]; }; } const coalesceBoundClassAliases = ( candidates: readonly AddressedName[], - bindings: readonly ObjcClassImportBinding[], + bindings: readonly MachoImportBinding[], ): ObjcClassOutputEntry[] => { const bindingsByAddress = new Map( bindings.map((binding) => [binding.requested_address, binding]), ); - const bindingKey = (binding: ObjcClassImportBinding) => + const bindingKey = (binding: MachoImportBinding) => JSON.stringify([ binding.symbol, binding.binding_source, @@ -271,7 +268,7 @@ const coalesceBoundClassAliases = ( binding.weak, binding.addend, ]); - const groupedBindings = new Map(); + const groupedBindings = new Map(); for (const binding of bindings) { const key = bindingKey(binding); const existing = groupedBindings.get(key); diff --git a/src/native/AppleDispatchCoverage.ts b/src/native/AppleDispatchCoverage.ts index 44c6b9f369..3d5ef4afd3 100644 --- a/src/native/AppleDispatchCoverage.ts +++ b/src/native/AppleDispatchCoverage.ts @@ -4,19 +4,22 @@ import type { DecodeIssue } from "./AppleDispatchDecodeFacts.js"; import type { FacetDecodeFacts } from "./AppleDispatchDecodeFacts.js"; const describeFixups = (fixups: PointerFixups): string => { - const unread = - fixups.unreadRebaseBytes > 0 - ? `LC_DYLD_INFO rebase opcodes were not decoded (${fixups.unreadRebaseBytes} bytes)` - : undefined; - const kind = - fixups.kind === "chained" - ? `chained fixups: ${fixups.formats.join(", ") || "no fixup segments"}` - : fixups.kind === "dyld-info" - ? "LC_DYLD_INFO bind opcodes" - : unread === undefined - ? "no fixup load commands" - : undefined; - return [unread, kind].filter((line) => line !== undefined).join("; "); + const descriptions: string[] = []; + if (fixups.unreadRebaseBytes > 0) + descriptions.push( + `LC_DYLD_INFO rebase opcodes were not decoded (${fixups.unreadRebaseBytes} bytes)`, + ); + if (fixups.kind === "chained") + descriptions.push( + `chained fixups: ${fixups.formats.join(", ") || "no fixup segments"}`, + ); + if (fixups.legacyRebaseStreams > 0) + descriptions.push("LC_DYLD_INFO rebase opcodes"); + if (fixups.legacyBindStreams > 0) + descriptions.push("LC_DYLD_INFO bind opcodes"); + return descriptions.length === 0 + ? "no fixup load commands" + : descriptions.join("; "); }; /** Append the per-facet coverage of one Apple dispatch metadata decode. */ diff --git a/src/native/AppleDispatchMetadata.test.ts b/src/native/AppleDispatchMetadata.test.ts index 6a46b9bac8..92250d71bc 100644 --- a/src/native/AppleDispatchMetadata.test.ts +++ b/src/native/AppleDispatchMetadata.test.ts @@ -120,6 +120,236 @@ const fixture = ( }; const provenance = { path: "/fixture/app", sha256: "a".repeat(64) }; +const swiftExternalSwiftBindingImage = ( + boundField: "protocol" | "type" = "protocol", +) => { + const bytes = fixture(); + const sectionCommand = 32; + bytes.writeUInt32LE(232, sectionCommand + 4); + bytes.writeUInt32LE(2, sectionCommand + 64); + bytes.write("__swift5_proto", 184); + bytes.write("__DATA", 200); + bytes.writeBigUInt64LE(base + 0xa00n, 216); + bytes.writeBigUInt64LE(4n, 224); + bytes.writeUInt32LE(0xa00, 232); + bytes.writeUInt32LE(2, 236); + + const dyldInfo = sectionCommand + 232; + bytes.writeUInt32LE(0x80000022, dyldInfo); + bytes.writeUInt32LE(48, dyldInfo + 4); + bytes.writeUInt32LE(0xf00, dyldInfo + 16); + const bind = Buffer.from([ + 0x11, // SET_DYLIB_ORDINAL_IMM 1 + 0x40, + ...Buffer.from(`${boundField === "protocol" ? "_$sSQMp" : "_$sSiMn"}\0`), + 0x70, + 0x80, + 0x16, // SET_SEGMENT_AND_OFFSET_ULEB segment 0 + offset 0xb00 + 0x90, + 0x00, + ]); + bytes.writeUInt32LE(bind.length, dyldInfo + 20); + const dylibCommand = dyldInfo + 48; + bytes.writeUInt32LE(0xc, dylibCommand); // LC_LOAD_DYLIB + bytes.writeUInt32LE(56, dylibCommand + 4); + bytes.writeUInt32LE(24, dylibCommand + 8); + bytes.write("/usr/lib/libswiftCore.dylib\0", dylibCommand + 24); + bytes.writeUInt32LE(3, 16); // segment, dyld info, and dependent dylib + bytes.writeUInt32LE(336, 20); + bind.copy(bytes, 0xf00); + + // Conformance at 0xa10: one reference is an indirect relative pointer to + // the import slot at 0xb00; the sibling reference points to local metadata. + bytes.writeInt32LE(0x10, 0xa00); + bytes.writeInt32LE(boundField === "protocol" ? 0x1ec : 0xec, 0xa14); + bytes.writeInt32LE(0, 0xa18); // Nullable witness-table pattern. + bytes.writeUInt32LE(boundField === "protocol" ? 0 : 8, 0xa1c); + bytes.writeInt32LE(boundField === "protocol" ? 0xf1 : 0x2f0, 0xa10); // Protocol reference: imported indirect slot or local descriptor. + bytes.writeBigUInt64LE(0n, 0xb00); // Bound slot has no in-image pointer. + bytes.writeUInt32LE(16, 0xc00); + bytes.writeInt32LE(0x1f8, 0xc08); + if (boundField === "type") { + bytes.writeUInt32LE(3, 0xd00); + bytes.writeInt32LE(0xf8, 0xd08); + bytes.write("Equatable\0", 0xe00); + } else { + bytes.write("Widget\0", 0xe00); + } + return bytes; +}; + +const swiftBoundWitnessSlotImage = () => { + const bytes = fixture(); + const sectionCommand = 32; + bytes.writeUInt32LE(232, sectionCommand + 4); + bytes.writeUInt32LE(2, sectionCommand + 64); + bytes.write("__swift5_proto", 184); + bytes.write("__DATA", 200); + bytes.writeBigUInt64LE(base + 0xa00n, 216); + bytes.writeBigUInt64LE(4n, 224); + bytes.writeUInt32LE(0xa00, 232); + bytes.writeUInt32LE(2, 236); + + const dyldInfo = sectionCommand + 232; + bytes.writeUInt32LE(0x80000022, dyldInfo); + bytes.writeUInt32LE(48, dyldInfo + 4); + bytes.writeUInt32LE(0xf20, dyldInfo + 8); + bytes.writeUInt32LE(6, dyldInfo + 12); + bytes.writeUInt32LE(0xf00, dyldInfo + 16); + const bind = Buffer.from([ + 0x11, + 0x40, + ...Buffer.from("_externalWitnessImpl\0"), + 0x70, + 0x88, + 0x1c, // SET_SEGMENT_AND_OFFSET_ULEB segment 0 + witness slot 0xe08 + 0x90, + 0x00, + ]); + bytes.writeUInt32LE(bind.length, dyldInfo + 20); + const dylibCommand = dyldInfo + 48; + bytes.writeUInt32LE(0xc, dylibCommand); + bytes.writeUInt32LE(56, dylibCommand + 4); + bytes.writeUInt32LE(24, dylibCommand + 8); + bytes.write("/usr/lib/libswiftCore.dylib\0", dylibCommand + 24); + bytes.writeUInt32LE(3, 16); + bytes.writeUInt32LE(336, 20); + bind.copy(bytes, 0xf00); + // One unrelated, valid local rebase proves unmatched structural pointers + // stay known-raw after the complete legacy stream is walked. + Buffer.from([0x11, 0x20, 0xa0, 0x1c, 0x51, 0]).copy(bytes, 0xf20); + + bytes.writeInt32LE(0x10, 0xa00); + bytes.writeInt32LE(0x1ec, 0xa14); + bytes.writeInt32LE(0x3e8, 0xa18); + bytes.writeUInt32LE(0, 0xa1c); + bytes.writeInt32LE(0x2f0, 0xa10); + bytes.writeUInt32LE(16, 0xc00); + bytes.writeInt32LE(0x238, 0xc08); + bytes.writeUInt32LE(3, 0xd00); + bytes.writeInt32LE(0x148, 0xd08); + bytes.writeUInt32LE(0, 0xd0c); + bytes.writeUInt32LE(1, 0xd10); + bytes.writeUInt32LE(1, 0xd18); + bytes.writeBigUInt64LE(base + 0xa10n, 0xe00); + bytes.write("Widget\0", 0xe40); + bytes.write("Equatable\0", 0xe50); + return bytes; +}; + +describe("Swift imported protocol references", () => { + it("preserves bind identity without reading VM address zero and marks coverage partial", () => { + const result = decodeAppleDispatchMetadata( + swiftExternalSwiftBindingImage(), + 100, + provenance, + ); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: "Widget", + protocol_name: null, + protocol_binding: { + symbol: "_$sSQMp", + requested_address: "0x100000b00", + }, + decode: { status: "partial" }, + }, + ]); + expect( + result.relative_pointers.find( + ({ field }) => field.address === "0x100000a10", + ), + ).toMatchObject({ + target: { address: null, file_offset: null }, + binding: { symbol: "_$sSQMp" }, + decode: { status: "partial" }, + }); + expect( + result.coverage.find( + ({ facet }) => facet === "swift_conformances_static_witness_slots", + ), + ).toMatchObject({ status: "partial" }); + }); + + it("preserves an indirect imported type reference as a typed binding", () => { + const result = decodeAppleDispatchMetadata( + swiftExternalSwiftBindingImage("type"), + 100, + provenance, + ); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: null, + type_binding: { + symbol: "_$sSiMn", + requested_address: "0x100000b00", + }, + protocol_name: "Equatable", + protocol_binding: null, + decode: { status: "partial" }, + }, + ]); + expect( + result.coverage.find( + ({ facet }) => facet === "swift_conformances_static_witness_slots", + ), + ).toMatchObject({ status: "partial" }); + }); + + it("reports only the legacy stream mechanisms actually examined", () => { + const bytes = swiftBoundWitnessSlotImage(); + const dyldInfo = 32 + 232; + bytes.writeUInt32LE(0, dyldInfo + 16); + bytes.writeUInt32LE(0, dyldInfo + 20); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + expect( + result.coverage.find(({ facet }) => facet === "pointer_fixups"), + ).toEqual({ + facet: "pointer_fixups", + status: "complete", + reason: "LC_DYLD_INFO rebase opcodes", + examined: 1, + decoded: 1, + }); + }); + + it("retains an unmatched local witness header beside a legacy rebase and preserves a bound implementation", () => { + const result = decodeAppleDispatchMetadata( + swiftBoundWitnessSlotImage(), + 100, + provenance, + ); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: "Widget", + protocol_name: "Equatable", + witness_table: { address: "0x100000e00" }, + decode: { status: "decoded" }, + }, + ]); + expect(result.swift_dispatch_slots).toMatchObject([ + { + table_kind: "witness_table", + slot_index: 1, + implementation_address: null, + implementation_binding: { + symbol: "_externalWitnessImpl", + requested_address: "0x100000e08", + }, + decode: { status: "partial" }, + }, + ]); + expect( + result.coverage.find( + ({ facet }) => facet === "swift_conformances_static_witness_slots", + ), + ).toMatchObject({ status: "partial" }); + }); +}); + const withProtocolListSection = (bytes: Buffer): Buffer => { const pointer = (at: number, offset: number) => bytes.writeBigUInt64LE(base + BigInt(offset), at); @@ -254,6 +484,37 @@ describe("Apple dispatch pointer fixup metadata", () => { }); }); describe("Apple dispatch binary metadata", () => { + it("retains structural Objective-C records when legacy rebase coverage is unknown", () => { + const bytes = fixture(); + // Add LC_DYLD_INFO_ONLY with a nonempty rebase stream. Legacy rebase + // stream is malformed, so provenance is unknown while readable + // nonzero structural pointer words must remain useful. + bytes.writeUInt32LE(2, 16); + bytes.writeUInt32LE(200, 20); + bytes.writeUInt32LE(0x80000022, 184); + bytes.writeUInt32LE(48, 188); + bytes.writeUInt32LE(4000, 192); + bytes.writeUInt32LE(1, 196); + bytes.writeUInt8(0xf0, 4000); + + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + expect(result.objc_classes[0]?.name).toBe("Fixture"); + expect(result.objc_classes[0]?.methods).toMatchObject([ + { selector: "performAction:" }, + ]); + expect(result.objc_classes[0]?.decode).toMatchObject({ + status: "partial", + }); + expect( + result.coverage.find( + ({ facet }) => facet === "objc_class_method_ivar_metadata", + ), + ).toMatchObject({ status: "partial" }); + expect( + result.coverage.find(({ facet }) => facet === "pointer_fixups"), + ).toMatchObject({ status: "partial" }); + }); + it.each([false, true])( "decodes stripped absolute/relative method lists (relative=%s) with exact offsets", (relative) => { diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index dfcbb5f924..007e44f8b5 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -76,10 +76,18 @@ export const decodeAppleDispatchMetadata = ( const u32 = (address: bigint) => bytes.readUInt32LE(offset(address, 4)); const decodePointer = (address: bigint) => fixups.decode(address, bytes.readBigUInt64LE(offset(address, 8))); - /** Rebased target of a pointer; a bind to another image reads as 0. */ + /** In-image target of a raw or rebased pointer; a bind is not an address. */ const pointer = (address: bigint) => { const decodedPointer = decodePointer(address); - return decodedPointer.kind === "rebase" ? decodedPointer.target : 0n; + if (decodedPointer.kind === "rebase" || decodedPointer.kind === "raw") + return decodedPointer.target; + if (decodedPointer.kind === "bind") return 0n; + // Preserve the on-disk observation for structural Objective-C metadata. + // `raw` is not promoted to an authoritative fixup result: the shared + // pointer-fixup coverage remains partial, and consumers that need to + // distinguish null from an unresolved bind use resolvePointer directly. + if (decodedPointer.kind === "unknown") return decodedPointer.raw; + throw new RangeError(decodedPointer.reason); }; const bound = (address: bigint) => { const decodedPointer = decodePointer(address); @@ -152,6 +160,7 @@ export const decodeAppleDispatchMetadata = ( i32: (address: bigint) => bytes.readInt32LE(offset(address, 4)), u64: (address: bigint) => bytes.readBigUInt64LE(offset(address, 8)), pointer, + resolvePointer: decodePointer, bound, string, location, diff --git a/src/native/AppleMachoFixups.test.ts b/src/native/AppleMachoFixups.test.ts index 8653b73e74..41f9c8b4ed 100644 --- a/src/native/AppleMachoFixups.test.ts +++ b/src/native/AppleMachoFixups.test.ts @@ -155,6 +155,347 @@ const SEGMENTS: Segment[] = [ { address: BASE + 0x4000n, size: 0x4000n, offset: 0x4000, executable: false }, ]; +const legacyRebaseInput = (stream: Buffer) => { + const data = Buffer.alloc(0x8000); + stream.copy(data, 0x7000); + return { + data, + fixups: parsePointerFixups( + data, + { + chained: null, + binds: [], + rebases: [{ offset: 0x7000, size: stream.length }], + dylibs: [], + }, + [{ address: BASE, size: 0x2000n, offset: 0, executable: false }], + BASE, + ), + }; +}; + +it("marks only opcode-selected pointer slots rebased, including target zero", () => { + // SET_TYPE(pointer), SET_SEGMENT(offset 0x100), DO once, + // ADD_ADDR(0x10), DO once, DONE. + const { data, fixups } = legacyRebaseInput( + Buffer.from([0x11, 0x20, 0x80, 0x02, 0x51, 0x30, 0x10, 0x51, 0]), + ); + data.writeBigUInt64LE(BASE + 0x200n, 0x100); + data.writeBigUInt64LE(0n, 0x118); + + expect(fixups.decode(BASE + 0x100n, BASE + 0x200n)).toEqual({ + kind: "rebase", + target: BASE + 0x200n, + }); + expect(fixups.decode(BASE + 0x118n, 0n)).toEqual({ + kind: "rebase", + target: 0n, + }); + expect(fixups.decode(BASE + 0x110n, 0n)).toEqual({ + kind: "raw", + target: 0n, + }); + expect(fixups).toMatchObject({ + kind: "dyld-info", + examined: 1, + decoded: 1, + unreadRebaseBytes: 0, + }); +}); + +it("applies count-and-skip opcodes to their emitted pointer locations", () => { + // SET_TYPE(pointer), SET_SEGMENT(offset 0x100), DO 1 skipping 8, + // DO_REBASE_ADD_ADDR(8), DO once, DONE. The selected slots are 0x100, 0x110, + // and 0x120; the skipped words remain raw. + const { fixups } = legacyRebaseInput( + Buffer.from([ + 0x11, 0x20, 0x80, 0x02, 0x80, 0x01, 0x08, 0x70, 0x08, 0x51, 0, + ]), + ); + + expect(fixups.decode(BASE + 0x100n, 0x200n)).toEqual({ + kind: "rebase", + target: 0x200n, + }); + expect(fixups.decode(BASE + 0x110n, 0x210n)).toEqual({ + kind: "rebase", + target: 0x210n, + }); + expect(fixups.decode(BASE + 0x120n, 0x220n)).toEqual({ + kind: "rebase", + target: 0x220n, + }); + expect(fixups.decode(BASE + 0x108n, 0n)).toEqual({ + kind: "raw", + target: 0n, + }); + expect(fixups.decode(BASE + 0x118n, 0n)).toEqual({ + kind: "raw", + target: 0n, + }); +}); + +it("retains known prefix rebases and leaves unmatched locations unknown on an invalid tail", () => { + const { fixups } = legacyRebaseInput( + Buffer.from([0x11, 0x20, 0x80, 0x02, 0x51, 0xf0]), + ); + + expect(fixups.decode(BASE + 0x100n, BASE + 0x200n)).toEqual({ + kind: "rebase", + target: BASE + 0x200n, + }); + expect(fixups.decode(BASE + 0x108n, BASE + 0x300n)).toMatchObject({ + kind: "unknown", + raw: BASE + 0x300n, + reason: expect.stringContaining("Unsupported rebase opcode 0xf0"), + }); + expect(fixups.issues).toMatchObject([ + { code: "rebase_stream_decode_failed" }, + ]); +}); + +it("does not treat 32-bit text rebases as 64-bit metadata pointers", () => { + const { fixups } = legacyRebaseInput( + Buffer.from([0x12, 0x20, 0x80, 0x02, 0x51, 0]), + ); + + expect(fixups.decode(BASE + 0x100n, BASE + 0x200n)).toMatchObject({ + kind: "unsupported", + reason: "Legacy non-pointer rebase is not a 64-bit metadata pointer", + }); + expect(fixups.decode(BASE + 0x108n, BASE + 0x300n)).toMatchObject({ + kind: "raw", + target: BASE + 0x300n, + }); +}); + +it("retains unaligned pointer locations accepted by dyld's rebase state checks", () => { + const { fixups } = legacyRebaseInput( + Buffer.from([0x11, 0x20, 0x81, 0x02, 0x51, 0]), + ); + + expect(fixups.decode(BASE + 0x101n, 0x1234n)).toEqual({ + kind: "rebase", + target: 0x1234n, + }); + expect(fixups.decode(BASE + 0x100n, 0n)).toEqual({ + kind: "raw", + target: 0n, + }); +}); + +it("rejects count expansion outside the selected file-backed segment without expanding it", () => { + const { fixups } = legacyRebaseInput( + Buffer.from([0x11, 0x20, 0x00, 0x60, 0x80, 0x80, 0x80, 0x80, 0x10, 0x00]), + ); + + expect(fixups.unreadRebaseBytes).toBeGreaterThan(0); + expect(fixups.decode(BASE + 0x100n, 0n)).toMatchObject({ + kind: "unknown", + reason: expect.stringContaining("leaves its file-backed segment"), + }); +}); + +it("bounds cumulative work across repeated full-range runs and retains the prefix", () => { + const data = Buffer.alloc(0x10000); + const repeatedRun = Buffer.from([ + 0x11, // SET_TYPE(pointer) + 0x20, + 0, // SET_SEGMENT_AND_OFFSET(0, 0) + 0x60, + 0x80, + 0x40, // DO_REBASE_ULEB_TIMES(8192) + ]); + const stream = Buffer.concat([ + ...Array.from({ length: 100 }, () => repeatedRun), + Buffer.from([0]), + ]); + stream.copy(data); + const fixups = parsePointerFixups( + data, + { + chained: null, + binds: [], + rebases: [{ offset: 0, size: stream.length }], + dylibs: [], + }, + [ + { + address: BASE, + size: BigInt(data.length), + offset: 0, + executable: false, + }, + ], + BASE, + ); + + expect(fixups.decoded).toBe(0); + expect(fixups.failures.join("\n")).toContain("source-derived budget"); + expect(fixups.decode(BASE + 0x100n, 0x1234n)).toEqual({ + kind: "rebase", + target: 0x1234n, + }); + expect(fixups.decode(BASE + 0x101n, 0n)).toMatchObject({ + kind: "unknown", + raw: 0n, + }); +}); + +it("shares the expansion budget across separate rebase streams", () => { + const data = Buffer.alloc(0x10000); + const stream = Buffer.from([0x11, 0x20, 0, 0x60, 0x80, 0x40, 0]); + const controlData = Buffer.alloc(0x10000); + stream.copy(controlData); + const singleRun = parsePointerFixups( + controlData, + { + chained: null, + binds: [], + rebases: [{ offset: 0, size: stream.length }], + dylibs: [], + }, + [ + { + address: BASE, + size: BigInt(controlData.length), + offset: 0, + executable: false, + }, + ], + BASE, + ); + expect(singleRun).toMatchObject({ examined: 1, decoded: 1, failures: [] }); + + const rebases = Array.from({ length: 17 }, (_, index) => { + const offset = index * 16; + stream.copy(data, offset); + return { offset, size: stream.length }; + }); + const fixups = parsePointerFixups( + data, + { + chained: null, + binds: [], + rebases, + dylibs: [], + }, + [ + { + address: BASE, + size: BigInt(data.length), + offset: 0, + executable: false, + }, + ], + BASE, + ); + + expect(fixups).toMatchObject({ examined: 17, decoded: 15 }); + expect(fixups.failures.join("\n")).toContain("source-derived budget"); + expect(fixups.decode(BASE + 0x100n, 0x1234n)).toEqual({ + kind: "rebase", + target: 0x1234n, + }); +}); + +describe("incomplete legacy bind applicability", () => { + it("keeps unmatched and known-rebase slots unknown when a bind tail is malformed", () => { + const data = Buffer.alloc(0x2000); + const bind = Buffer.from([0x10, 0x40, 0x78, 0, 0x70, 0, 0xf0, 0x90]); + const rebase = Buffer.from([0x11, 0x20, 0x80, 0x20, 0x51, 0]); + bind.copy(data, 0); + rebase.copy(data, 16); + const fixups = parsePointerFixups( + data, + { + chained: null, + binds: [{ offset: 0, size: bind.length, stream: "bind" }], + rebases: [{ offset: 16, size: rebase.length }], + dylibs: [], + }, + [ + { + address: BASE, + size: BigInt(data.length), + offset: 0, + executable: false, + }, + ], + BASE, + ); + + expect(fixups.decode(BASE + 0x1000n, 0n)).toMatchObject({ + kind: "unknown", + raw: 0n, + reason: expect.stringContaining("could overlap this rebase location"), + }); + expect(fixups.decode(BASE + 0x1008n, 0n)).toMatchObject({ + kind: "unknown", + raw: 0n, + reason: expect.stringContaining("pointer-fixup applicability"), + }); + expect(fixups.issues).toMatchObject([ + { code: "bind_stream_decode_failed" }, + ]); + }); + + it("retains a positive bind prefix while leaving unmatched slots unknown", () => { + const opcodes = Buffer.from([ + 0x11, + 0x40, + ...Buffer.from("_x\0"), + 0x71, + 0x08, + 0x90, + 0xf0, + ]); + const fixups = parsePointerFixups( + opcodes, + { + chained: null, + binds: [{ offset: 0, size: opcodes.length, stream: "bind" }], + rebases: [], + dylibs: ["/library-1"], + }, + SEGMENTS, + BASE, + ); + + expect(fixups.decode(BASE + 0x4008n, 0n)).toMatchObject({ + kind: "bind", + symbol: "_x", + }); + expect(fixups.decode(BASE + 0x4010n, 0n)).toMatchObject({ + kind: "unknown", + raw: 0n, + }); + }); + + it("keeps unmatched chained-image slots unknown when a legacy bind tail fails", () => { + const chained = chainedData(1); + const bind = Buffer.from([0xf0, 0x90]); + const data = Buffer.concat([chained, bind]); + const fixups = parsePointerFixups( + data, + { + chained: { offset: 0, size: chained.length }, + binds: [{ offset: chained.length, size: bind.length, stream: "bind" }], + rebases: [], + dylibs: ["/library-1", "/library-2"], + }, + SEGMENTS, + BASE, + ); + + expect(fixups.decode(BASE + 0x4010n, 0n)).toMatchObject({ + kind: "unknown", + raw: 0n, + reason: expect.stringContaining("pointer-fixup applicability"), + }); + }); +}); + describe("independent chained segment coverage", () => { it("counts successfully decoded chained segments alongside an unsupported segment", () => { const starts = 28; @@ -232,13 +573,14 @@ describe("pointer fixup tables", () => { }); it("keeps unread LC_DYLD_INFO rebase bytes when chained fixups are present", () => { - const data = chainedData(1); + const chained = chainedData(1); + const data = Buffer.concat([chained, Buffer.from([0xf0])]); const fixups = parsePointerFixups( data, { - chained: { offset: 0, size: data.length }, + chained: { offset: 0, size: chained.length }, binds: [], - rebases: [{ offset: 8, size: 12 }], + rebases: [{ offset: chained.length, size: 1 }], dylibs: ["/library-1", "/library-2"], }, SEGMENTS, @@ -246,7 +588,7 @@ describe("pointer fixup tables", () => { ); expect(fixups).toMatchObject({ kind: "chained", - unreadRebaseBytes: 12, + unreadRebaseBytes: 1, }); }); @@ -260,7 +602,7 @@ describe("pointer fixup tables", () => { { chained: { offset: 0, size: 128 }, binds: [], - rebases: [{ offset: 8, size: 12 }], + rebases: [], dylibs: ["/library-1", "/library-2"], }, SEGMENTS, @@ -269,7 +611,7 @@ describe("pointer fixup tables", () => { expect(fixups).toMatchObject({ examined: 1, decoded: 1, - unreadRebaseBytes: 12, + unreadRebaseBytes: 0, }); expect(fixups.decode(BASE + 0x4008n, field(1n, 63))).toMatchObject({ kind: "bind", @@ -313,7 +655,7 @@ describe("pointer fixup tables", () => { }); // Pointers outside fixup segments are stored plainly. expect(fixups.decode(BASE + 0x10n, 0x1234n)).toEqual({ - kind: "rebase", + kind: "raw", target: 0x1234n, }); }); @@ -381,7 +723,7 @@ describe("parsed chained pointer membership", () => { ); const plain = data.readBigUInt64LE(0x400b); expect(fixups.decode(BASE + 0x400bn, plain)).toEqual({ - kind: "rebase", + kind: "raw", target: plain, }); }); @@ -664,11 +1006,11 @@ describe("chained pointer membership", () => { symbol: "_free", }); expect(fixups.decode(BASE + 0x4010n, plain)).toEqual({ - kind: "rebase", + kind: "raw", target: plain, }); expect(fixups.decode(BASE + 0x4030n, 0n)).toEqual({ - kind: "rebase", + kind: "raw", target: 0n, }); }, @@ -693,11 +1035,11 @@ describe("chained pointer membership", () => { data.writeUInt16LE(0x1000, 44); const unknown = parsePointerFixups(data, commands, SEGMENTS, BASE); expect(unknown.decode(BASE + 0x4008n, BASE + 8n)).toEqual({ - kind: "rebase", + kind: "raw", target: BASE + 8n, }); expect(unknown.decode(BASE + 0x5008n, BASE + 16n)).toEqual({ - kind: "rebase", + kind: "raw", target: BASE + 16n, }); }); @@ -940,7 +1282,7 @@ describe("legacy pointer fixup streams", () => { library: "/usr/lib/libSystem.B.dylib", }); expect(fixups.decode(BASE + 0x4018n, 0x99n)).toEqual({ - kind: "rebase", + kind: "raw", target: 0x99n, }); }); @@ -981,7 +1323,7 @@ describe("legacy pointer fixup streams", () => { library: "/usr/lib/libobjc.A.dylib", }); expect(fixups.decode(BASE + 0x4018n, 0x99n)).toEqual({ - kind: "rebase", + kind: "raw", target: 0x99n, }); }); diff --git a/src/native/AppleMachoFixups.ts b/src/native/AppleMachoFixups.ts index 50bb241477..78de93e90c 100644 --- a/src/native/AppleMachoFixups.ts +++ b/src/native/AppleMachoFixups.ts @@ -37,6 +37,18 @@ export type DecodedPointer = } | { readonly kind: "unsupported"; readonly reason: string }; +/** Pointer-slot outcome, including whether a rebase was actually applied. */ +export type PointerResolution = + | { readonly kind: "raw"; readonly target: bigint } + | { readonly kind: "rebase"; readonly target: bigint } + | Exclude + | { + readonly kind: "unknown"; + /** Preserved on-disk word; not an authoritative in-image target. */ + readonly raw: bigint; + readonly reason: string; + }; + /** Pointer decoding for one Mach-O slice. */ export interface PointerFixups { readonly kind: "chained" | "dyld-info" | "none"; @@ -44,14 +56,17 @@ export interface PointerFixups { readonly formats: readonly string[]; readonly failures: readonly string[]; readonly issues: readonly DecodeIssue[]; - /** Fixup segments or bind streams this reader walked. */ + /** Chained segments, legacy rebase streams, or bind streams this reader walked. */ readonly examined: number; /** Fully decoded fixup segments or bind streams, in the same units as `examined`. */ readonly decoded: number; - /** Rebase opcode bytes present in the load command and not decoded. */ + /** Declared legacy opcode streams examined by this owner. */ + readonly legacyRebaseStreams: number; + readonly legacyBindStreams: number; + /** Bytes in rebase streams that could not be completely decoded. */ readonly unreadRebaseBytes: number; /** Decode the pointer stored at `address` whose raw 64-bit value is `raw`. */ - decode(address: bigint, raw: bigint): DecodedPointer; + decode(address: bigint, raw: bigint): PointerResolution; } const POINTER_FORMATS: Readonly> = { @@ -86,6 +101,15 @@ interface BindRun { | { readonly kind: "unsupported"; readonly reason: string }; } +interface BindFacts { + readonly runs: readonly BindRun[]; + readonly failures: readonly string[]; + readonly issues: readonly DecodeIssue[]; + readonly examined: number; + readonly decoded: number; + readonly complete: boolean; +} + /** Resolve only valid dylib ordinals; preserve special scopes as raw ordinals. */ const libraryName = ( ordinal: number, @@ -283,7 +307,8 @@ const readChainedPage = ( const chainedFixups = ( image: FixupImage, range: { readonly offset: number; readonly size: number }, - unreadRebaseBytes: number, + rebases: RebaseFacts, + binds: BindFacts, ): PointerFixups => { const { bytes, segments, dylibs, baseAddress } = image; const base = range.offset; @@ -412,18 +437,22 @@ const chainedFixups = ( ranges.map(({ format }) => POINTER_FORMATS[format] ?? `format-${format}`), ), ]; - const issues = ranges.flatMap(({ format, segment }) => - POINTER_FORMATS[format] === undefined - ? [ - issue( - "pointer_fixups", - "unsupported_pointer_format", - `Unsupported chained pointer format ${format}`, - `0x${segment.address.toString(16)}`, - ), - ] - : [], - ); + const issues = [ + ...rebases.issues, + ...binds.issues, + ...ranges.flatMap(({ format, segment }) => + POINTER_FORMATS[format] === undefined + ? [ + issue( + "pointer_fixups", + "unsupported_pointer_format", + `Unsupported chained pointer format ${format}`, + `0x${segment.address.toString(16)}`, + ), + ] + : [], + ), + ]; const examinedViews = new Map(); const decodedRanges = new Set(); for (const [chainIndex, chain] of ranges.entries()) { @@ -461,17 +490,58 @@ const chainedFixups = ( if (complete) decodedRanges.add(chainIndex); } const failures = issues.map(({ message }) => message); + const applicabilityFailures = [...rebases.failures, ...binds.failures]; + const applicabilityComplete = rebases.complete && binds.complete; return { kind: "chained", formats, failures, issues, - examined: Math.max(ranges.length, 1), + examined: Math.max(ranges.length, 1) + rebases.examined + binds.examined, get decoded() { - return ranges.length === 0 ? 1 : decodedRanges.size; + return ( + (ranges.length === 0 ? 1 : decodedRanges.size) + + rebases.decoded + + binds.decoded + ); }, - unreadRebaseBytes, + legacyRebaseStreams: rebases.examined, + legacyBindStreams: binds.examined, + unreadRebaseBytes: rebases.unreadBytes, decode(address, raw) { + const rawPointer = (): PointerResolution => + applicabilityComplete + ? { kind: "raw", target: raw } + : { + kind: "unknown", + raw, + reason: `LC_DYLD_INFO pointer-fixup applicability was not completely decoded: ${applicabilityFailures.join("; ")}`, + }; + const legacyBind = resolveLegacyBind(binds.runs, address); + const legacyRebase = resolveLegacyRebase( + segments, + rebases.membership, + address, + raw, + ); + if (legacyBind?.kind === "unsupported") return legacyBind; + if (legacyBind !== undefined && legacyRebase !== undefined) + return { + kind: "unsupported", + reason: `Both bind and rebase records target pointer location 0x${address.toString(16)}`, + }; + if (legacyBind !== undefined && !rebases.complete) + return { + kind: "unknown", + raw, + reason: `Legacy rebase opcodes could overlap this bind location: ${rebases.failures.join("; ")}`, + }; + if (legacyRebase !== undefined && !binds.complete) + return { + kind: "unknown", + raw, + reason: `Legacy bind opcodes could overlap this rebase location: ${binds.failures.join("; ")}`, + }; const chainIndex = ranges.findIndex( ({ segment }) => address >= segment.address && @@ -479,12 +549,14 @@ const chainedFixups = ( ); const chain = ranges[chainIndex]; // Pointers outside fixup segments, such as in __TEXT, are stored plainly. - if (chain === undefined) return { kind: "rebase", target: raw }; + if (chain === undefined) + return legacyBind ?? legacyRebase ?? rawPointer(); const within = Number(address - chain.segment.address); const page = Math.floor(within / chain.pageSize); - if (page >= chain.pageCount) return { kind: "rebase", target: raw }; + if (page >= chain.pageCount) + return legacyBind ?? legacyRebase ?? rawPointer(); const start = bytes.readUInt16LE(chain.pageStarts + page * 2); - if (start === 0xffff) return { kind: "rebase", target: raw }; + if (start === 0xffff) return legacyBind ?? legacyRebase ?? rawPointer(); if (POINTER_FORMATS[chain.format] === undefined) return { kind: "unsupported", @@ -494,14 +566,37 @@ const chainedFixups = ( const facts = pageFacts(chain, page, start); const at = within % chain.pageSize; const locations = facts.locations; - if (((locations[at >>> 3] ?? 0) & (1 << (at & 7))) !== 0) - return decodeChainedPointer(raw, chain.format, baseAddress, imports); + if (((locations[at >>> 3] ?? 0) & (1 << (at & 7))) !== 0) { + if ( + legacyBind !== undefined || + legacyRebase !== undefined || + !applicabilityComplete + ) { + if (legacyBind !== undefined || legacyRebase !== undefined) + return { + kind: "unsupported", + reason: `Both chained and legacy fixup records target pointer location 0x${address.toString(16)}`, + }; + return { + kind: "unknown", + raw, + reason: `Incomplete legacy fixup streams could overlap chained pointer location 0x${address.toString(16)}: ${applicabilityFailures.join("; ")}`, + }; + } + const decoded = decodeChainedPointer( + raw, + chain.format, + baseAddress, + imports, + ); + return decoded; + } if ( facts.failure !== undefined && (facts.lastOffset === undefined || at > facts.lastOffset) ) throw new RangeError(facts.failure); - return { kind: "rebase", target: raw }; + return legacyBind ?? legacyRebase ?? rawPointer(); } catch (cause: unknown) { const reason = cause instanceof Error ? cause.message : String(cause); const location = `0x${address.toString(16)}`; @@ -529,15 +624,29 @@ const chainedFixups = ( }; }; -const uleb = (bytes: Buffer, cursor: { at: number }, end: number): bigint => { +const uleb = ( + bytes: Buffer, + cursor: { at: number }, + end: number, + stream: "bind" | "rebase" = "bind", + rebaseWork?: { workRemaining: number }, +): bigint => { let value = 0n; let shift = 0; for (;;) { - if (cursor.at >= end) throw new RangeError("Truncated bind opcode operand"); + if (cursor.at >= end) + throw new RangeError(`Truncated ${stream} opcode operand`); + if (rebaseWork !== undefined) { + if (rebaseWork.workRemaining === 0) + throw new RangeError( + "Legacy rebase parsing exceeded the source-derived work budget", + ); + rebaseWork.workRemaining--; + } const byte = bytes[cursor.at++] ?? 0; const slice = byte & 0x7f; if ((shift === 63 && slice > 1) || (shift === 64 && slice !== 0)) - throw new RangeError("Oversized bind opcode operand"); + throw new RangeError(`Oversized ${stream} opcode operand`); if (shift < 64) value |= BigInt(slice) << BigInt(shift); // Padding may extend the encoding, but never the represented integer. shift = Math.min(64, shift + 7); @@ -545,6 +654,235 @@ const uleb = (bytes: Buffer, cursor: { at: number }, end: number): bigint => { } }; +interface RebaseFacts { + /** Two-bit state per source byte; allocated lazily only for emitted rebases. */ + readonly membership: Uint8Array | null; + readonly failures: readonly string[]; + readonly issues: readonly DecodeIssue[]; + readonly examined: number; + readonly decoded: number; + readonly unreadBytes: number; + readonly complete: boolean; +} + +const REBASE_POINTER_SIZE = 8n; +const REBASE_POINTER = 1; +const REBASE_NON_POINTER = 2; +const REBASE_CONFLICT = 3; + +const markRebaseLocation = ( + index: { bitmap: Uint8Array | null; workRemaining: number }, + fileOffset: number, + type: number, + fileBytes: number, +): void => { + index.bitmap ??= new Uint8Array(Math.ceil(fileBytes / 4)); + const at = Math.floor(fileOffset / 4); + const shift = (fileOffset % 4) * 2; + const previous = ((index.bitmap[at] ?? 0) >> shift) & 3; + const next = previous === 0 || previous === type ? type : REBASE_CONFLICT; + index.bitmap[at] = + ((index.bitmap[at] ?? 0) & ~(3 << shift)) | (next << shift); +}; + +/** Interpret one bounded LC_DYLD_INFO rebase stream into file-backed membership. */ +const readRebaseStream = ( + image: FixupImage, + range: FixupCommands["rebases"][number], + index: { bitmap: Uint8Array | null; workRemaining: number }, +): void => { + const { bytes, segments } = image; + const end = range.offset + range.size; + if ( + !Number.isSafeInteger(range.offset) || + !Number.isSafeInteger(range.size) || + range.offset < 0 || + range.size < 0 || + !Number.isSafeInteger(end) || + end > bytes.length + ) + throw new RangeError("Rebase opcodes exceed the target"); + const cursor = { at: range.offset }; + let segmentIndex = -1; + let segmentOffset = 0n; + let type = 0; + const addRun = (count: bigint, skip: bigint): void => { + if (count === 0n) return; + if (type === 0 || type > 3) + throw new RangeError(`Unsupported legacy rebase type ${type}`); + const segment = segments[segmentIndex]; + if (segment === undefined) + throw new RangeError("Rebase location has no selected segment"); + const stride = REBASE_POINTER_SIZE + skip; + const lastOffset = segmentOffset + (count - 1n) * stride; + if (segmentOffset < 0n || lastOffset + REBASE_POINTER_SIZE > segment.size) + throw new RangeError("Rebase expansion leaves its file-backed segment"); + if (!Number.isSafeInteger(segment.offset) || segment.offset < 0) + throw new RangeError("Rebase segment file offset is invalid"); + const firstFileOffset = BigInt(segment.offset) + segmentOffset; + if ( + firstFileOffset < 0n || + lastOffset + BigInt(segment.offset) + REBASE_POINTER_SIZE > + BigInt(bytes.length) + ) + throw new RangeError("Rebase location exceeds the source file"); + const fact = type === 1 ? REBASE_POINTER : REBASE_NON_POINTER; + const admitted = + count < BigInt(index.workRemaining) ? count : BigInt(index.workRemaining); + for (let indexOffset = 0n; indexOffset < admitted; indexOffset++) { + const fileOffset = Number(firstFileOffset + indexOffset * stride); + markRebaseLocation(index, fileOffset, fact, bytes.length); + } + index.workRemaining -= Number(admitted); + if (admitted < count) + throw new RangeError( + "Legacy rebase opcode and expansion work exceeded the source-derived budget", + ); + // dyld advances after the final record too; that cursor may legally be + // one past the segment because it is only bounds-checked on the next emit. + segmentOffset += count * stride; + }; + while (cursor.at < end) { + if (index.workRemaining === 0) + throw new RangeError( + "Legacy rebase parsing exceeded the source-derived work budget", + ); + index.workRemaining--; + const byte = bytes[cursor.at++] ?? 0; + const immediate = byte & 0x0f; + switch (byte & 0xf0) { + case 0x00: // REBASE_OPCODE_DONE; up to 15 trailing bytes are alignment. + if (end - cursor.at > 15) + throw new RangeError( + "Rebase opcodes terminated before the stream end", + ); + cursor.at = end; + break; + case 0x10: // REBASE_OPCODE_SET_TYPE_IMM + type = immediate; + break; + case 0x20: { + // REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB + if (segments[immediate] === undefined) + throw new RangeError("Rebase segment index is out of range"); + segmentIndex = immediate; + segmentOffset = uleb(bytes, cursor, end, "rebase", index); + break; + } + case 0x30: // REBASE_OPCODE_ADD_ADDR_ULEB + segmentOffset += uleb(bytes, cursor, end, "rebase", index); + break; + case 0x40: // REBASE_OPCODE_ADD_ADDR_IMM_SCALED + segmentOffset += BigInt(immediate) * REBASE_POINTER_SIZE; + break; + case 0x50: // REBASE_OPCODE_DO_REBASE_IMM_TIMES + addRun(BigInt(immediate), 0n); + break; + case 0x60: { + // REBASE_OPCODE_DO_REBASE_ULEB_TIMES + const count = uleb(bytes, cursor, end, "rebase", index); + addRun(count, 0n); + break; + } + case 0x70: { + // REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB + addRun(1n, 0n); + segmentOffset += uleb(bytes, cursor, end, "rebase", index); + break; + } + case 0x80: { + // REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB + const count = uleb(bytes, cursor, end, "rebase", index); + const skip = uleb(bytes, cursor, end, "rebase", index); + addRun(count, skip); + break; + } + default: + throw new RangeError( + `Unsupported rebase opcode 0x${byte.toString(16)}`, + ); + } + } +}; + +const resolveLegacyRebase = ( + segments: readonly Segment[], + membership: Uint8Array | null, + address: bigint, + raw: bigint, +): PointerResolution | undefined => { + if (membership === null) return undefined; + let fileOffset: number | undefined; + for (const segment of segments) { + const relative = address - segment.address; + if (relative < 0n || relative + REBASE_POINTER_SIZE > segment.size) + continue; + if (fileOffset !== undefined) + return { + kind: "unsupported", + reason: `Ambiguous segments map legacy rebase pointer location 0x${address.toString(16)}`, + }; + const mapped = BigInt(segment.offset) + relative; + if (mapped < 0n || mapped > BigInt(Number.MAX_SAFE_INTEGER)) + return { + kind: "unsupported", + reason: "Legacy rebase file offset exceeds the safe integer range", + }; + fileOffset = Number(mapped); + } + if (fileOffset === undefined) return undefined; + const at = Math.floor(fileOffset / 4); + const shift = (fileOffset % 4) * 2; + const state = ((membership[at] ?? 0) >> shift) & 3; + if (state === 0) return undefined; + if (state === REBASE_POINTER) return { kind: "rebase", target: raw }; + return { + kind: "unsupported", + reason: + state === REBASE_CONFLICT + ? `Conflicting legacy rebase kinds target pointer location 0x${address.toString(16)}` + : "Legacy non-pointer rebase is not a 64-bit metadata pointer", + }; +}; + +const parseLegacyRebases = ( + image: FixupImage, + streams: FixupCommands["rebases"], +): RebaseFacts => { + // Dyld does not require pointer alignment here, so up to one starting byte + // per source byte is possible. Bound opcode scanning plus slot visits by 2N. + const workBudget = Math.min(Number.MAX_SAFE_INTEGER, image.bytes.length * 2); + const index: { bitmap: Uint8Array | null; workRemaining: number } = { + bitmap: null, + workRemaining: workBudget, + }; + const failures: string[] = []; + let decoded = 0; + let unreadBytes = 0; + for (const stream of streams) { + try { + readRebaseStream(image, stream, index); + decoded++; + } catch (cause: unknown) { + failures.push( + `rebase opcodes: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + unreadBytes += stream.size; + } + } + return { + membership: index.bitmap, + failures, + issues: failures.map((message) => + issue("pointer_fixups", "rebase_stream_decode_failed", message), + ), + examined: streams.length, + decoded, + unreadBytes, + complete: failures.length === 0, + }; +}; + const sleb = (bytes: Buffer, cursor: { at: number }, end: number): bigint => { let value = 0n; let shift = 0; @@ -759,6 +1097,62 @@ const readBindStream = ( } }; +const resolveLegacyBind = ( + runs: readonly BindRun[], + address: bigint, +): PointerResolution | undefined => { + let match: BindRun | undefined; + for (const run of runs) { + const distance = address - run.start; + if ( + distance >= 0n && + distance % run.stride === 0n && + distance / run.stride < run.count + ) { + if (match !== undefined) + return { + kind: "unsupported", + reason: `Ambiguous bind records target pointer location 0x${address.toString(16)}`, + }; + match = run; + } + } + return match === undefined + ? undefined + : match.binding.kind === "unsupported" + ? match.binding + : { kind: "bind", ...match.binding.value }; +}; + +const parseLegacyBinds = ( + image: FixupImage, + streams: FixupCommands["binds"], +): BindFacts => { + const runs: BindRun[] = []; + const failures: string[] = []; + let decoded = 0; + for (const stream of streams) { + try { + readBindStream(image, stream, runs); + decoded++; + } catch (cause: unknown) { + failures.push( + `${stream.stream} bind opcodes: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + } + return { + runs, + failures, + issues: failures.map((message) => + issue("pointer_fixups", "bind_stream_decode_failed", message), + ), + examined: streams.length, + decoded, + complete: failures.length === 0, + }; +}; + /** Build pointer decoding for one slice from its fixup load commands. */ export const parsePointerFixups = ( bytes: Buffer, @@ -767,14 +1161,22 @@ export const parsePointerFixups = ( baseAddress: bigint, ): PointerFixups => { const image = { bytes, segments, dylibs: commands.dylibs, baseAddress }; - const unreadRebaseBytes = commands.rebases.reduce( - (total, rebase) => total + rebase.size, - 0, - ); + const rebases = parseLegacyRebases(image, commands.rebases); + const binds = parseLegacyBinds(image, commands.binds); if (commands.chained !== null) { - return chainedFixups(image, commands.chained, unreadRebaseBytes); + return chainedFixups(image, commands.chained, rebases, binds); } - if (commands.binds.length === 0) + const applicabilityFailures = [...rebases.failures, ...binds.failures]; + const applicabilityComplete = rebases.complete && binds.complete; + const rawPointer = (raw: bigint): PointerResolution => + applicabilityComplete + ? { kind: "raw", target: raw } + : { + kind: "unknown", + raw, + reason: `LC_DYLD_INFO pointer-fixup applicability was not completely decoded: ${applicabilityFailures.join("; ")}`, + }; + if (commands.binds.length === 0 && commands.rebases.length === 0) return { kind: "none", formats: [], @@ -782,54 +1184,51 @@ export const parsePointerFixups = ( issues: [], examined: 0, decoded: 0, - unreadRebaseBytes, - decode: (_address, raw) => ({ kind: "rebase", target: raw }), + legacyRebaseStreams: 0, + legacyBindStreams: 0, + unreadRebaseBytes: 0, + decode: (_address, raw) => rawPointer(raw), }; - const binds: BindRun[] = []; - const failures: string[] = []; - let decoded = 0; - for (const stream of commands.binds) - try { - readBindStream(image, stream, binds); - decoded++; - } catch (cause: unknown) { - failures.push( - `${stream.stream} bind opcodes: ${cause instanceof Error ? cause.message : String(cause)}`, - ); - } + const failures = [...rebases.failures, ...binds.failures]; + const issues = [...rebases.issues, ...binds.issues]; return { kind: "dyld-info", formats: [], failures, - issues: failures.map((message) => - issue("pointer_fixups", "bind_stream_decode_failed", message), - ), - examined: commands.binds.length, - decoded, - unreadRebaseBytes, + issues, + examined: binds.examined + rebases.examined, + decoded: binds.decoded + rebases.decoded, + legacyRebaseStreams: rebases.examined, + legacyBindStreams: binds.examined, + unreadRebaseBytes: rebases.unreadBytes, decode(address, raw) { - let match: BindRun | undefined; - for (const run of binds) { - const { start, count, stride } = run; - const distance = address - start; - if ( - distance >= 0n && - distance % stride === 0n && - distance / stride < count - ) { - if (match !== undefined) - return { - kind: "unsupported", - reason: `Ambiguous bind records target pointer location 0x${address.toString(16)}`, - }; - match = run; - } - } - return match === undefined - ? { kind: "rebase", target: raw } - : match.binding.kind === "unsupported" - ? match.binding - : { kind: "bind", ...match.binding.value }; + const match = resolveLegacyBind(binds.runs, address); + const rebase = resolveLegacyRebase( + segments, + rebases.membership, + address, + raw, + ); + if (match?.kind === "unsupported") return match; + if (match !== undefined && rebase !== undefined) + return { + kind: "unsupported", + reason: `Both bind and rebase records target pointer location 0x${address.toString(16)}`, + }; + if (match !== undefined && !rebases.complete) + return { + kind: "unknown", + raw, + reason: `Legacy rebase opcodes could overlap this bind location: ${rebases.failures.join("; ")}`, + }; + if (match !== undefined) return match; + if (rebase !== undefined && !binds.complete) + return { + kind: "unknown", + raw, + reason: `Legacy bind opcodes could overlap this rebase location: ${binds.failures.join("; ")}`, + }; + return rebase ?? rawPointer(raw); }, }; }; diff --git a/src/native/AppleObjcClassImportBindings.test.ts b/src/native/AppleObjcClassImportBindings.test.ts index 89cf06bcfa..f63330a4aa 100644 --- a/src/native/AppleObjcClassImportBindings.test.ts +++ b/src/native/AppleObjcClassImportBindings.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import type { PointerFixups } from "./AppleMachoFixups.js"; +import { parsePointerFixups } from "./AppleMachoFixups.js"; import type { MachoLayout } from "./AppleMachoSelection.js"; import { inspectObjcClassImportBindings } from "./AppleObjcClassImportBindings.js"; @@ -15,6 +16,8 @@ const createInput = ( issues: [], examined: 1, decoded: 1, + legacyRebaseStreams: 0, + legacyBindStreams: 0, unreadRebaseBytes: 0, decode: (address) => address === 0x1000n @@ -133,6 +136,39 @@ describe("Objective-C import binding address inspection", () => { }); }); + it("marks selected slots partial when legacy rebase ownership is unknown", () => { + const input = createInput(); + const streams = Buffer.from([0x20, 0x80, 0]); // Truncated rebase operand; valid bind DONE. + const fixups = parsePointerFixups( + streams, + { + chained: null, + binds: [{ offset: 2, size: 1, stream: "bind" }], + rebases: [{ offset: 0, size: 2 }], + dylibs: [], + }, + [{ address: 0x1000n, size: 0x1000n, offset: 0, executable: false }], + 0n, + ); + expect( + inspectObjcClassImportBindings({ + ...input, + addresses: ["0x1000"], + fixups, + }), + ).toMatchObject({ + bindings: [], + coverage: { + status: "partial", + reason: expect.stringContaining( + "LC_DYLD_INFO pointer-fixup applicability was not completely decoded", + ), + examined: 1, + decoded: 0, + }, + }); + }); + it("does not infer absent imports from unsupported fixup coverage", () => { const input = createInput({ fixups: { ...createInput().fixups, kind: "none" }, diff --git a/src/native/AppleObjcClassImportBindings.ts b/src/native/AppleObjcClassImportBindings.ts index d6fa3fcdce..e55eea7257 100644 --- a/src/native/AppleObjcClassImportBindings.ts +++ b/src/native/AppleObjcClassImportBindings.ts @@ -1,9 +1,9 @@ import type { PointerFixups } from "./AppleMachoFixups.js"; import type { MachoLayout } from "./AppleMachoSelection.js"; -import type { ObjcClassImportBinding } from "../domain/native/objcSwiftMetadata.js"; +import type { MachoImportBinding } from "../domain/native/objcSwiftMetadata.js"; export interface ObjcClassImportBindingScan { - readonly bindings: readonly ObjcClassImportBinding[]; + readonly bindings: readonly MachoImportBinding[]; readonly coverage: { readonly status: "complete" | "partial" | "unsupported" | "not_requested"; readonly reason: string | null; @@ -51,7 +51,7 @@ export const inspectObjcClassImportBindings = (input: { ); for (const failure of fixups.failures) reasons.add(failure); - const bindings: ObjcClassImportBinding[] = []; + const bindings: MachoImportBinding[] = []; const examinedAddresses = new Set(); const candidates = addresses.slice(0, maxRecords); if (candidates.length < addresses.length) @@ -76,6 +76,10 @@ export const inspectObjcClassImportBindings = (input: { reasons.add(`${requestedAddress}: ${decoded.reason}`); continue; } + if (decoded.kind === "unknown") { + reasons.add(`${requestedAddress}: ${decoded.reason}`); + continue; + } if (decoded.kind !== "bind" || !isObjcClassImport(decoded.symbol)) continue; bindings.push({ diff --git a/src/native/AppleSwiftDispatchFacets.ts b/src/native/AppleSwiftDispatchFacets.ts index c22683a9f8..3c41cf7007 100644 --- a/src/native/AppleSwiftDispatchFacets.ts +++ b/src/native/AppleSwiftDispatchFacets.ts @@ -1,4 +1,5 @@ import type { ObjcSwiftMetadata } from "../domain/native/objcSwiftMetadata.js"; +import type { PointerResolution } from "./AppleMachoFixups.js"; import type { Segment, Section } from "./AppleMachoSelection.js"; import { issue, type FacetDecodeFacts } from "./AppleDispatchDecodeFacts.js"; @@ -25,10 +26,69 @@ export interface DispatchRecordBudget { admit(): boolean; } +/** Field-owned Swift ABI pointer encodings used by the readers below. */ +export type SwiftRelativeField = + | "conformanceRecord" + | "conformanceProtocol" + | "protocolName" + | "conformanceType" + | "typeName" + | "witnessTablePattern" + | "typeMetadataRecord" + | "classDescriptorName" + | "methodImplementation"; + +type SwiftRelativeFieldContract = { + encoding: "direct" | "indirectable" | "taggedDirect" | "compactFunction"; + nullability: "nullable" | "nonnull"; +}; + +/** Nullability and storage encoding follow the owning fields in Swift ABI. */ +const relativeFieldContracts = { + conformanceRecord: { encoding: "direct", nullability: "nonnull" }, + conformanceProtocol: { + encoding: "indirectable", + nullability: "nullable", + }, + protocolName: { encoding: "direct", nullability: "nonnull" }, + conformanceType: { encoding: "direct", nullability: "nullable" }, + typeName: { encoding: "direct", nullability: "nonnull" }, + witnessTablePattern: { encoding: "direct", nullability: "nullable" }, + typeMetadataRecord: { encoding: "taggedDirect", nullability: "nonnull" }, + classDescriptorName: { encoding: "direct", nullability: "nonnull" }, + methodImplementation: { + encoding: "compactFunction", + nullability: "nullable", + }, +} as const satisfies Record; + +/** Resolved address or a null encoding permitted by its Swift ABI field. */ +export type SwiftRelativeTarget = bigint | null; +export type SwiftRelativeValue = + | { readonly kind: "target"; readonly target: bigint } + | { readonly kind: "null" } + | { + readonly kind: "binding"; + readonly binding: Extract; + readonly requestedAddress: bigint; + } + | { readonly kind: "unknown"; readonly reason: string }; + +/** Keep ABI null, unavailable values, and unresolved imports distinct. */ +export const unresolvedSwiftRelativeReason = ( + value: Exclude, + field: SwiftRelativeField, +): string => { + if (value.kind === "null") return `Null Swift relative pointer for ${field}`; + if (value.kind === "unknown") return value.reason; + return `Swift relative pointer for ${field} has an unresolved external binding at ${hex(value.requestedAddress)}`; +}; + /** Byte readers for Swift relative-pointer and conformance decode facets. */ export interface SwiftDispatchReaders { u32(address: bigint): number; pointer(address: bigint): bigint; + resolvePointer(address: bigint): PointerResolution; string(address: bigint): string; location(address: bigint): ResolvedMetadataLocation; evidence(address: bigint, description: string): BinaryMetadataEvidence; @@ -37,6 +97,18 @@ export interface SwiftDispatchReaders { const hex = (value: bigint) => `0x${value.toString(16)}`; const decoded = { status: "decoded" as const, reason: null }; +const importBinding = ( + requestedAddress: bigint, + binding: Extract, +) => ({ + requested_address: hex(requestedAddress), + symbol: binding.symbol, + binding_source: binding.binding_source, + library_ordinal: binding.library_ordinal, + library: binding.library, + weak: binding.weak, + addend: binding.addend.toString(), +}); /** Create a Swift ABI signed-relative pointer decoder that records relative_pointers. */ export const createSwiftRelativeReader = (input: { @@ -46,18 +118,93 @@ export const createSwiftRelativeReader = (input: { result: ObjcSwiftMetadata; }) => { const { bytes, readers: read, budget, result } = input; - return (field: bigint, indirectable = false): bigint => { + return (field: bigint, kind: SwiftRelativeField): SwiftRelativeValue => { + const contract = relativeFieldContracts[kind]; + const indirectable = contract.encoding === "indirectable"; + const taggedDirect = contract.encoding === "taggedDirect"; + const nullable = contract.nullability === "nullable"; const displacement = bytes.readInt32LE(read.offset(field, 4)); + const encodingTag = taggedDirect ? displacement & 3 : undefined; + if (encodingTag !== undefined && encodingTag !== 0) { + const reason = `Unsupported Swift type metadata record tag ${encodingTag}`; + if (budget.admit()) + result.relative_pointers.push({ + field: read.location(field), + target: { address: null, file_offset: null }, + displacement, + indirectable, + indirect: false, + encoding_tag: encodingTag, + decode: { status: "unsupported", reason }, + evidence: read.evidence( + field, + "Swift ABI tagged type metadata record", + ), + }); + return { kind: "unknown", reason }; + } const indirect = indirectable && (displacement & 1) !== 0; - let target = - displacement === 0 - ? 0n - : field + BigInt(indirectable ? displacement & ~1 : displacement); + let target: SwiftRelativeTarget = + nullable && displacement === 0 + ? null + : field + + BigInt( + taggedDirect + ? displacement & ~3 + : indirectable + ? displacement & ~1 + : displacement, + ); let fileOffset: number | null = null; let reason: string | null = null; try { - if (indirect && target !== 0n) target = read.pointer(target); - if (target !== 0n) fileOffset = read.offset(target); + if (indirect && target !== null) { + const pointer = read.resolvePointer(target); + if (pointer.kind === "bind") { + if (budget.admit()) + result.relative_pointers.push({ + field: read.location(field), + target: { address: null, file_offset: null }, + displacement, + indirectable, + indirect, + binding: importBinding(target, pointer), + decode: { status: "partial", reason: "External pointer binding" }, + evidence: read.evidence( + field, + "Swift ABI indirect relative pointer and Mach-O import binding", + ), + }); + return { + kind: "binding", + binding: pointer, + requestedAddress: target, + }; + } + if (pointer.kind === "unknown" || pointer.kind === "unsupported") { + const unresolved = pointer.reason; + if (budget.admit()) + result.relative_pointers.push({ + field: read.location(field), + target: { address: null, file_offset: null }, + displacement, + indirectable, + indirect, + decode: { status: "partial", reason: unresolved }, + evidence: read.evidence( + field, + "Swift ABI indirect relative pointer", + ), + }); + return { kind: "unknown", reason: unresolved }; + } + const indirectTarget = pointer.target; + target = + pointer.kind === "raw" && indirectTarget === 0n + ? null + : indirectTarget; + } + if (target !== null) fileOffset = read.offset(target); } catch (cause) { reason = cause instanceof Error ? cause.message : String(cause); } @@ -65,20 +212,20 @@ export const createSwiftRelativeReader = (input: { result.relative_pointers.push({ field: read.location(field), target: { - address: target === 0n ? null : hex(target), + address: target === null ? null : hex(target), file_offset: fileOffset, }, displacement, indirectable, indirect, + ...(encodingTag === undefined ? {} : { encoding_tag: encodingTag }), decode: reason === null ? decoded : { status: "partial", reason }, evidence: read.evidence( field, "Swift ABI signed-relative metadata pointer", ), }); - if (reason !== null) throw new RangeError(reason); - return target; + return target === null ? { kind: "null" } : { kind: "target", target }; }; }; @@ -87,13 +234,22 @@ export const decodeSwiftDispatchFacets = (input: { sections: readonly Section[]; segments: readonly Segment[]; readers: SwiftDispatchReaders; - relative: (field: bigint, indirectable?: boolean) => bigint; + relative: (field: bigint, kind: SwiftRelativeField) => SwiftRelativeValue; budget: DispatchRecordBudget; result: ObjcSwiftMetadata; }): FacetDecodeFacts => { const { sections, segments, readers: read, relative, budget, result } = input; const swiftFailures: string[] = []; const issues = [] as ReturnType[]; + const requiredRelative = ( + field: bigint, + kind: SwiftRelativeField, + ): bigint => { + const value = relative(field, kind); + if (value.kind !== "target") + throw new RangeError(unresolvedSwiftRelativeReason(value, kind)); + return value.target; + }; let swiftExamined = 0; let decodedRecords = 0; for (const section of sections.filter( @@ -106,43 +262,113 @@ export const decodeSwiftDispatchFacets = (input: { swiftExamined++; const field = section.address + BigInt(index * 4); try { - const descriptor = relative(field); + const descriptor = requiredRelative(field, "conformanceRecord"); const flags = read.u32(descriptor + 12n); - const protocol = relative(descriptor, true); - if ((read.u32(protocol) & 31) !== 3) + const protocolValue = relative(descriptor, "conformanceProtocol"); + const protocol = + protocolValue.kind === "target" ? protocolValue.target : null; + const unresolvedProtocolReason = + protocolValue.kind === "unknown" || protocolValue.kind === "binding" + ? unresolvedSwiftRelativeReason( + protocolValue, + "conformanceProtocol", + ) + : null; + if (protocol !== null && (read.u32(protocol) & 31) !== 3) throw new TypeError("Unsupported Swift protocol descriptor kind"); - const protocolName = read.string(relative(protocol + 8n)); + const protocolName = + protocol === null + ? null + : read.string(requiredRelative(protocol + 8n, "protocolName")); + const protocolBinding = + protocolValue.kind === "binding" + ? importBinding( + protocolValue.requestedAddress, + protocolValue.binding, + ) + : null; const typeKind = (flags >>> 3) & 7; if (typeKind !== 0 && typeKind !== 1) throw new TypeError( "Unsupported Swift conformance type-reference kind", ); - let type = relative(descriptor + 4n); - if (typeKind === 1) type = read.pointer(type); - if (![16, 17, 18].includes(read.u32(type) & 31)) + const typeValue = relative(descriptor + 4n, "conformanceType"); + let type = typeValue.kind === "target" ? typeValue.target : null; + let typeBinding = + typeValue.kind === "binding" + ? importBinding(typeValue.requestedAddress, typeValue.binding) + : null; + let unresolvedTypeReason = + typeValue.kind === "unknown" ? typeValue.reason : null; + if (typeKind === 1 && type !== null) { + const pointer = read.resolvePointer(type); + if (pointer.kind === "bind") { + typeBinding = importBinding(type, pointer); + type = null; + } else if (pointer.kind === "raw" && pointer.target === 0n) { + type = null; + unresolvedTypeReason = + "Null indirect Swift conformance type reference"; + } else if (pointer.kind === "rebase" || pointer.kind === "raw") { + type = pointer.target; + } else { + type = null; + unresolvedTypeReason = pointer.reason; + } + } + if (type !== null && ![16, 17, 18].includes(read.u32(type) & 31)) throw new TypeError("Unsupported Swift type descriptor kind"); - const typeName = read.string(relative(type + 8n)); - const witness = relative(descriptor + 8n); + const typeName = + type === null + ? null + : read.string(requiredRelative(type + 8n, "typeName")); + const witnessValue = + protocol !== null && type !== null + ? relative(descriptor + 8n, "witnessTablePattern") + : { kind: "null" as const }; + const witness = + witnessValue.kind === "target" ? witnessValue.target : null; + const unresolvedWitnessReason = + witnessValue.kind === "unknown" || witnessValue.kind === "binding" + ? unresolvedSwiftRelativeReason(witnessValue, "witnessTablePattern") + : null; const staticWitness = - witness !== 0n && + witness !== null && + protocol !== null && + type !== null && (flags & ~0x38) === 0 && (read.u32(type) & 0x80) === 0; let witnessReason: string | null = staticWitness ? null - : "Conditional, generic, resilient or missing static witness table is not decoded"; - if (staticWitness && read.pointer(witness) !== descriptor) - witnessReason = - "Witness-table conformance header is unresolved or encoded"; + : (unresolvedProtocolReason ?? + unresolvedTypeReason ?? + unresolvedWitnessReason ?? + (protocolBinding !== null || typeBinding !== null + ? "Conformance references an external Mach-O binding" + : "Conditional, generic, resilient or missing static witness table is not decoded")); + if (staticWitness) { + const header = read.resolvePointer(witness); + if (header.kind !== "raw" && header.kind !== "rebase") + witnessReason = + header.kind === "unknown" || header.kind === "unsupported" + ? header.reason + : "Witness-table conformance header is an external binding"; + else if (header.target !== descriptor) + witnessReason = + "Witness-table conformance header is unresolved or encoded"; + } const conformanceDecode = witnessReason === null ? { status: "decoded" as const, reason: null } : { status: "partial" as const, reason: witnessReason }; result.swift_conformances.push({ type_name: typeName, + type_binding: typeBinding, protocol_name: protocolName, + protocol_binding: protocolBinding, module: null, witness_table: - witness === 0n + witness === null ? { address: null, file_offset: null } : read.location(witness), location: read.location(descriptor), @@ -153,33 +379,46 @@ export const decodeSwiftDispatchFacets = (input: { ), }); if (conformanceDecode.status === "decoded") decodedRecords++; - if (witnessReason !== null) { - swiftFailures.push(`${typeName}: ${witnessReason}`); + if ( + witnessReason !== null || + witness === null || + typeName === null || + protocolName === null + ) { + const reason = + witnessReason ?? "Missing static witness table is not decoded"; + swiftFailures.push(`${typeName ?? ""}: ${reason}`); issues.push( issue( "swift_conformances_static_witness_slots", "conformance_decode_partial", - witnessReason, + reason, hex(descriptor), ), ); continue; } - const count = read.u32(protocol + 16n); - const signatureCount = read.u32(protocol + 12n); + const count = read.u32(protocol! + 16n); + const signatureCount = read.u32(protocol! + 12n); if (count > 20_000 || signatureCount > 20_000) throw new RangeError("Swift witness requirement count exceeds 20000"); for (let slot = 0; slot < count; slot++) { if (!budget.admit()) break; const entry = witness + BigInt((slot + 1) * 8); - const implementation = read.pointer(entry); + const implementationResolution = read.resolvePointer(entry); + const implementation = + implementationResolution.kind === "raw" || + implementationResolution.kind === "rebase" + ? implementationResolution.target + : null; const requirementFlags = read.u32( - protocol + 24n + BigInt(signatureCount * 12 + slot * 8), + protocol! + 24n + BigInt(signatureCount * 12 + slot * 8), ); const kind = requirementFlags & 15; const synchronousFunction = kind >= 1 && kind <= 4 && (requirementFlags & 0x20) === 0; const executable = + implementation !== null && synchronousFunction && segments.some( (segment) => @@ -192,7 +431,12 @@ export const decodeSwiftDispatchFacets = (input: { : { status: "partial" as const, reason: - "Witness slot is non-function, async, nil, external, authenticated or encoded; implementation target is unresolved", + implementationResolution.kind === "bind" + ? "Witness implementation is an external Mach-O binding" + : implementationResolution.kind === "unknown" || + implementationResolution.kind === "unsupported" + ? implementationResolution.reason + : "Witness slot is non-function, async, nil, external, authenticated or encoded; implementation target is unresolved", }; result.swift_dispatch_slots.push({ owner: `${typeName}: ${protocolName}`, @@ -201,6 +445,14 @@ export const decodeSwiftDispatchFacets = (input: { requirement: null, implementation: null, implementation_address: executable ? hex(implementation) : null, + ...(implementationResolution.kind === "bind" + ? { + implementation_binding: importBinding( + entry, + implementationResolution, + ), + } + : {}), thunk_address: null, location: read.location(entry), decode: slotDecode, diff --git a/src/native/AppleSwiftRelativePointer.test.ts b/src/native/AppleSwiftRelativePointer.test.ts new file mode 100644 index 0000000000..e394a8ddd2 --- /dev/null +++ b/src/native/AppleSwiftRelativePointer.test.ts @@ -0,0 +1,274 @@ +import { expect, it } from "vitest"; + +import { objcSwiftMetadataSchema } from "../domain/native/objcSwiftMetadata.js"; +import { + createSwiftRelativeReader, + type DispatchRecordBudget, +} from "./AppleSwiftDispatchFacets.js"; + +const readers = (bytes: Buffer) => ({ + u32: (address: bigint) => bytes.readUInt32LE(Number(address)), + pointer: (address: bigint) => bytes.readBigUInt64LE(Number(address)), + resolvePointer: (address: bigint) => ({ + kind: "raw" as const, + target: bytes.readBigUInt64LE(Number(address)), + }), + string: () => "fixture", + location: (address: bigint) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), + evidence: (address: bigint, description: string) => [ + { + kind: "binary_metadata" as const, + description, + location: { + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }, + artifact_path: "/fixture", + artifact_sha256: "a".repeat(64), + }, + ], + offset: (address: bigint, size = 1) => { + if (address < 0n || address + BigInt(size) > BigInt(bytes.length)) + throw new RangeError("outside fixture bytes"); + return Number(address); + }, +}); + +const budget = (): DispatchRecordBudget => ({ + truncated: false, + isFull: () => false, + markTruncated: () => undefined, + admit: () => true, +}); + +it("keeps an encoded null distinct from a nonzero relative offset to address zero", () => { + const bytes = Buffer.alloc(0x80); + bytes.writeInt32LE(0, 0x20); + bytes.writeInt32LE(-0x40, 0x40); + bytes.writeInt32LE(0x25, 0x24); + bytes.writeBigUInt64LE(0n, 0x48); + bytes.writeInt32LE(0x25, 0x2c); + bytes.writeBigUInt64LE(0x60n, 0x50); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: readers(bytes), + budget: budget(), + result, + }); + + expect(relative(0x20n, "methodImplementation")).toEqual({ kind: "null" }); + expect(relative(0x40n, "methodImplementation")).toEqual({ + kind: "target", + target: 0n, + }); + expect(relative(0x24n, "conformanceProtocol")).toEqual({ kind: "null" }); + expect(relative(0x2cn, "conformanceProtocol")).toEqual({ + kind: "target", + target: 0x60n, + }); + expect(result.relative_pointers).toMatchObject([ + { displacement: 0, target: { address: null, file_offset: null } }, + { displacement: -0x40, target: { address: "0x0", file_offset: 0 } }, + { displacement: 0x25, indirect: true, target: { address: null } }, + { displacement: 0x25, indirect: true, target: { address: "0x60" } }, + ]); +}); + +it("resolves zero offsets for nonnullable fields to their own field address", () => { + const bytes = Buffer.alloc(0x80); + bytes.writeInt32LE(0, 0x20); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: readers(bytes), + budget: budget(), + result, + }); + + for (const kind of [ + "conformanceRecord", + "protocolName", + "typeName", + "typeMetadataRecord", + "classDescriptorName", + ] as const) { + expect(relative(0x20n, kind)).toEqual({ kind: "target", target: 0x20n }); + } + expect(result.relative_pointers).toHaveLength(5); + expect(result.relative_pointers).toMatchObject( + Array.from({ length: 5 }, () => ({ + target: { address: "0x20", file_offset: 0x20 }, + })), + ); +}); + +it("accepts encoded zero only for nullable direct Swift fields", () => { + const bytes = Buffer.alloc(0x80); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: readers(bytes), + budget: budget(), + result, + }); + + for (const kind of [ + "conformanceType", + "witnessTablePattern", + "methodImplementation", + ] as const) { + expect(relative(0x20n, kind)).toEqual({ kind: "null" }); + } +}); + +it("masks only the verified direct type metadata tag and keeps other tags unsupported", () => { + const bytes = Buffer.alloc(0x80); + bytes.writeInt32LE(0x24, 0x20); + bytes.writeInt32LE(0x25, 0x28); + bytes.writeInt32LE(0x26, 0x2c); + bytes.writeInt32LE(0x27, 0x30); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: readers(bytes), + budget: budget(), + result, + }); + + expect(relative(0x20n, "typeMetadataRecord")).toEqual({ + kind: "target", + target: 0x44n, + }); + for (const [field, tag] of [ + [0x28n, 1], + [0x2cn, 2], + [0x30n, 3], + ] as const) { + expect(relative(field, "typeMetadataRecord")).toEqual({ + kind: "unknown", + reason: `Unsupported Swift type metadata record tag ${tag}`, + }); + } + expect(result.relative_pointers.slice(-4)).toMatchObject([ + { displacement: 0x24, encoding_tag: 0, target: { address: "0x44" } }, + { + displacement: 0x25, + encoding_tag: 1, + decode: { status: "unsupported" }, + }, + { + displacement: 0x26, + encoding_tag: 2, + decode: { status: "unsupported" }, + }, + { + displacement: 0x27, + encoding_tag: 3, + decode: { status: "unsupported" }, + }, + ]); +}); + +it("treats a zero pointer loaded through a nullable indirect field as null", () => { + const bytes = Buffer.alloc(0x80); + bytes.writeInt32LE(-0x23, 0x24); // odd-tagged displacement to cell address 0 + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: readers(bytes), + budget: budget(), + result, + }); + + expect(relative(0x24n, "conformanceProtocol")).toEqual({ kind: "null" }); + expect(result.relative_pointers[0]).toMatchObject({ + indirect: true, + target: { address: null, file_offset: null }, + }); +}); + +it("keeps a fixup rebase to VM address zero as an address, not null", () => { + const bytes = Buffer.alloc(0x80); + bytes.writeInt32LE(-0x23, 0x24); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: { + ...readers(bytes), + resolvePointer: () => ({ kind: "rebase", target: 0n }), + offset: (address, size = 1) => { + if (address === 0n) throw new RangeError("unmapped target address"); + return readers(bytes).offset(address, size); + }, + }, + budget: budget(), + result, + }); + + expect(relative(0x24n, "conformanceProtocol")).toEqual({ + kind: "target", + target: 0n, + }); + expect(result.relative_pointers[0]).toMatchObject({ + target: { address: "0x0", file_offset: null }, + decode: { status: "partial", reason: "unmapped target address" }, + }); +}); + +it("preserves an indirect import binding instead of reading VM address zero", () => { + const bytes = Buffer.alloc(0x80); + bytes.writeInt32LE(-0x23, 0x24); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: { + ...readers(bytes), + resolvePointer: () => ({ + kind: "bind", + symbol: "_$sSQMp", + binding_source: "chained", + library_ordinal: 1, + library: "/System/Library/libswiftCore.dylib", + weak: false, + addend: 0n, + }), + }, + budget: budget(), + result, + }); + + expect(relative(0x24n, "conformanceProtocol")).toMatchObject({ + kind: "binding", + binding: { symbol: "_$sSQMp" }, + }); + expect(result.relative_pointers[0]).toMatchObject({ + target: { address: null, file_offset: null }, + binding: { symbol: "_$sSQMp", requested_address: "0x0" }, + decode: { status: "partial" }, + }); +}); + +it("retains unsafe target addresses as partial decode facts", () => { + const bytes = Buffer.alloc(0x80); + bytes.writeInt32LE(0x100, 0x20); + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const relative = createSwiftRelativeReader({ + bytes, + readers: readers(bytes), + budget: budget(), + result, + }); + + expect(relative(0x20n, "protocolName")).toEqual({ + kind: "target", + target: 0x120n, + }); + expect(result.relative_pointers[0]).toMatchObject({ + target: { address: "0x120", file_offset: null }, + decode: { status: "partial", reason: "outside fixture bytes" }, + }); +}); diff --git a/src/native/AppleSwiftVtables.test.ts b/src/native/AppleSwiftVtables.test.ts index 5873de9335..c1e5fb4b8f 100644 --- a/src/native/AppleSwiftVtables.test.ts +++ b/src/native/AppleSwiftVtables.test.ts @@ -7,140 +7,219 @@ import { decodeSwiftDispatchFacets } from "./AppleSwiftDispatchFacets.js"; import type { DispatchRecordBudget } from "./AppleSwiftDispatchFacets.js"; import { issue } from "./AppleDispatchDecodeFacts.js"; -describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => { - it("counts examined slots and reports an unresolved async slot", () => { - const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); - const values = new Map([ - [0x200n, 0x80000010], // class descriptor with a vtable - [0x22cn, 0], // table offset - [0x230n, 2], // slot count - [0x21cn, 2], // descriptor positive words - [0x234n, 0x40], // async method flag - ]); - const facts = decodeSwiftClassVtables({ - entries: [0x100n], - result, - readers: { - u32: (address) => values.get(address) ?? 0, - relative: (address) => - address === 0x100n ? 0x200n : address === 0x208n ? 0x300n : 0x400n, - string: () => "Widget", - location: (address) => ({ - address: `0x${address.toString(16)}`, - file_offset: Number(address), - }), - evidence: (address, description) => [ - { - kind: "binary_metadata", - description, - location: { - address: `0x${address.toString(16)}`, - file_offset: Number(address), - }, - artifact_path: "/fixture", - artifact_sha256: "a".repeat(64), - }, - ], - executable: (address) => address === 0x400n, - admit: () => true, - }, - }); - - expect(result.swift_dispatch_slots[0]?.decode.status).toBe("partial"); - expect(result.swift_dispatch_slots[1]?.decode.status).toBe("decoded"); - expect(facts).toMatchObject({ - facet: "swift_class_vtable_descriptors", - examined: 2, - decoded: 1, - exhaustive: false, - issues: [{ code: "vtable_implementation_unresolved", location: "0x234" }], - }); - - pushDispatchCoverage({ - result, - failures: [], - categoryIssues: [], - examined: 0, - categoriesExamined: 0, - truncated: false, - objcListScanTruncated: false, - objcClassListScanTruncated: false, - fixups: { - kind: "none", - formats: [], - failures: [], - issues: [], - decode: (_address, raw) => ({ kind: "rebase", target: raw }), - examined: 0, - decoded: 0, - unreadRebaseBytes: 0, - }, - swift: { - facet: "swift_conformances_static_witness_slots", - examined: 0, - decoded: 0, - exhaustive: true, - issues: [], - }, - vtables: facts, - }); - - expect( - result.coverage.find( - ({ facet }) => facet === "swift_class_vtable_descriptors", - ), - ).toMatchObject({ - status: "partial", - examined: 2, - decoded: 1, - reason: expect.stringContaining("0x234"), - }); +it("preserves unsupported type-metadata tag reasons instead of reporting null", () => { + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const facts = decodeSwiftClassVtables({ + entries: [0x100n], + result, + readers: { + u32: () => 0, + relative: () => ({ + kind: "unknown", + reason: "Unsupported Swift type metadata record tag 2", + }), + string: () => "unused", + location: (address) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), + evidence: () => [], + executable: () => false, + admit: () => true, + }, }); - it("returns a located issue for an emitted unresolved witness slot", () => { - const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); - const values = new Map([ - [0x20cn, 0], // simple, unconditional conformance - [0x500n, 3], // protocol descriptor kind - [0x50cn, 0], // signature count - [0x510n, 1], // witness requirement count - [0x2b0n, 16], // nominal type descriptor kind - [0x518n, 1], // function witness requirement - ]); - const budget: DispatchRecordBudget = { + expect(facts.issues).toMatchObject([ + { + code: "vtable_decode_failed", + message: "Unsupported Swift type metadata record tag 2", + }, + ]); + expect(facts.issues[0]?.message).not.toContain("Null Swift"); +}); + +it("preserves a required conformance pointer's unknown reason", () => { + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const facts = decodeSwiftDispatchFacets({ + sections: [{ name: "__swift5_proto", address: 0x100n, size: 4 }], + segments: [], + readers: { + u32: () => 0, + pointer: () => 0n, + resolvePointer: () => ({ kind: "raw", target: 0n }), + string: () => "unused", + location: (address) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), + evidence: () => [], + offset: (address) => Number(address), + }, + relative: () => ({ + kind: "unknown", + reason: "Unsupported Swift conformance record tag 3", + }), + budget: { truncated: false, isFull: () => false, - markTruncated: () => { - budget.truncated = true; - }, + markTruncated: () => {}, admit: () => true, - }; - const facts = decodeSwiftDispatchFacets({ - sections: [{ name: "__swift5_proto", address: 0x100n, size: 4 }], - segments: [], - readers: { - u32: (address) => values.get(address) ?? 0, - pointer: (address) => (address === 0x380n ? 0x200n : 0x900n), - string: (address) => (address === 0x600n ? "Proto" : "Widget"), - location: (address) => ({ - address: `0x${address.toString(16)}`, - file_offset: Number(address), - }), - evidence: (address, description) => [ - { - kind: "binary_metadata", - description, - location: { - address: `0x${address.toString(16)}`, - file_offset: Number(address), - }, - artifact_path: "/fixture", - artifact_sha256: "a".repeat(64), + }, + result, + }); + + expect(facts.issues).toMatchObject([ + { + code: "conformance_decode_failed", + message: "Unsupported Swift conformance record tag 3", + }, + ]); +}); + +it("counts examined slots and reports an unresolved async slot", () => { + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const values = new Map([ + [0x200n, 0x80000010], // class descriptor with a vtable + [0x22cn, 0], // table offset + [0x230n, 2], // slot count + [0x21cn, 2], // descriptor positive words + [0x234n, 0x40], // async method flag + ]); + const facts = decodeSwiftClassVtables({ + entries: [0x100n], + result, + readers: { + u32: (address) => values.get(address) ?? 0, + relative: (address) => ({ + kind: "target", + target: + address === 0x100n ? 0x200n : address === 0x208n ? 0x300n : 0x400n, + }), + string: () => "Widget", + location: (address) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), + evidence: (address, description) => [ + { + kind: "binary_metadata", + description, + location: { + address: `0x${address.toString(16)}`, + file_offset: Number(address), }, - ], - offset: (address) => Number(address), - }, - relative: (field) => + artifact_path: "/fixture", + artifact_sha256: "a".repeat(64), + }, + ], + executable: (address) => address === 0x400n, + admit: () => true, + }, + }); + + expect(result.swift_dispatch_slots[0]?.decode.status).toBe("partial"); + expect(result.swift_dispatch_slots[1]?.decode.status).toBe("decoded"); + expect(facts).toMatchObject({ + facet: "swift_class_vtable_descriptors", + examined: 2, + decoded: 1, + exhaustive: false, + issues: [{ code: "vtable_implementation_unresolved", location: "0x234" }], + }); + + pushDispatchCoverage({ + result, + failures: [], + categoryIssues: [], + examined: 0, + categoriesExamined: 0, + truncated: false, + objcListScanTruncated: false, + objcClassListScanTruncated: false, + fixups: { + kind: "none", + formats: [], + failures: [], + issues: [], + decode: (_address, raw) => ({ kind: "rebase", target: raw }), + examined: 0, + decoded: 0, + legacyRebaseStreams: 0, + legacyBindStreams: 0, + unreadRebaseBytes: 0, + }, + swift: { + facet: "swift_conformances_static_witness_slots", + examined: 0, + decoded: 0, + exhaustive: true, + issues: [], + }, + vtables: facts, + }); + + expect( + result.coverage.find( + ({ facet }) => facet === "swift_class_vtable_descriptors", + ), + ).toMatchObject({ + status: "partial", + examined: 2, + decoded: 1, + reason: expect.stringContaining("0x234"), + }); +}); + +it("returns a located issue for an emitted unresolved witness slot", () => { + const result = objcSwiftMetadataSchema.parse({ db_save_result: null }); + const values = new Map([ + [0x20cn, 0], // simple, unconditional conformance + [0x500n, 3], // protocol descriptor kind + [0x50cn, 0], // signature count + [0x510n, 1], // witness requirement count + [0x2b0n, 16], // nominal type descriptor kind + [0x518n, 1], // function witness requirement + ]); + const budget: DispatchRecordBudget = { + truncated: false, + isFull: () => false, + markTruncated: () => { + budget.truncated = true; + }, + admit: () => true, + }; + const facts = decodeSwiftDispatchFacets({ + sections: [{ name: "__swift5_proto", address: 0x100n, size: 4 }], + segments: [], + readers: { + u32: (address) => values.get(address) ?? 0, + pointer: (address) => (address === 0x380n ? 0x200n : 0x900n), + resolvePointer: (address) => ({ + kind: "raw", + target: address === 0x380n ? 0x200n : 0x900n, + }), + string: (address) => (address === 0x600n ? "Proto" : "Widget"), + location: (address) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), + evidence: (address, description) => [ + { + kind: "binary_metadata", + description, + location: { + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }, + artifact_path: "/fixture", + artifact_sha256: "a".repeat(64), + }, + ], + offset: (address) => Number(address), + }, + relative: (field) => ({ + kind: "target", + target: new Map([ [0x100n, 0x200n], [0x200n, 0x500n], @@ -149,16 +228,16 @@ describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => { [0x2b8n, 0x700n], [0x208n, 0x380n], ]).get(field) ?? 0n, - budget, - result, - }); + }), + budget, + result, + }); - expect(result.swift_dispatch_slots[0]?.decode.status).toBe("partial"); - expect(facts).toMatchObject({ - facet: "swift_conformances_static_witness_slots", - exhaustive: false, - issues: [{ code: "witness_slot_unresolved", location: "0x388" }], - }); + expect(result.swift_dispatch_slots[0]?.decode.status).toBe("partial"); + expect(facts).toMatchObject({ + facet: "swift_conformances_static_witness_slots", + exhaustive: false, + issues: [{ code: "witness_slot_unresolved", location: "0x388" }], }); }); @@ -192,6 +271,8 @@ describe("Swift vtable coverage facts: unsupported pointer fixup coverage", () = }), examined: 0, decoded: 0, + legacyRebaseStreams: 0, + legacyBindStreams: 0, unreadRebaseBytes: 0, }, swift: { diff --git a/src/native/AppleSwiftVtables.ts b/src/native/AppleSwiftVtables.ts index a680957ac2..fa563cb013 100644 --- a/src/native/AppleSwiftVtables.ts +++ b/src/native/AppleSwiftVtables.ts @@ -3,12 +3,17 @@ import type { NativeMetadataEvidence, NativeMetadataLocation, } from "../domain/native/objcSwiftMetadata.js"; +import type { + SwiftRelativeField, + SwiftRelativeValue, +} from "./AppleSwiftDispatchFacets.js"; +import { unresolvedSwiftRelativeReason } from "./AppleSwiftDispatchFacets.js"; import { issue, type FacetDecodeFacts } from "./AppleDispatchDecodeFacts.js"; /** Byte readers shared with the validated Mach-O metadata boundary. */ export interface SwiftMetadataReaders { u32(address: bigint): number; - relative(address: bigint): bigint; + relative(address: bigint, kind: SwiftRelativeField): SwiftRelativeValue; string(address: bigint): string; location(address: bigint): NativeMetadataLocation; evidence(address: bigint, description: string): NativeMetadataEvidence[]; @@ -43,7 +48,12 @@ export const decodeSwiftClassVtables = (input: { break; } try { - const descriptor = read.relative(field); + const descriptorValue = read.relative(field, "typeMetadataRecord"); + if (descriptorValue.kind !== "target") + throw new RangeError( + unresolvedSwiftRelativeReason(descriptorValue, "typeMetadataRecord"), + ); + const descriptor = descriptorValue.target; const flags = read.u32(descriptor); if ((flags & 31) !== 16) continue; const specific = flags >>> 16; @@ -66,7 +76,13 @@ export const decodeSwiftClassVtables = (input: { throw new RangeError( "Swift vtable count or offset exceeds declared metadata bounds", ); - const owner = read.string(read.relative(descriptor + 8n)); + const ownerValue = read.relative(descriptor + 8n, "classDescriptorName"); + if (ownerValue.kind !== "target") + throw new RangeError( + unresolvedSwiftRelativeReason(ownerValue, "classDescriptorName"), + ); + const ownerAddress = ownerValue.target; + const owner = read.string(ownerAddress); if (owner.length === 0) throw new RangeError( "Swift class vtable descriptor class name is empty", @@ -90,8 +106,25 @@ export const decodeSwiftClassVtables = (input: { // Async slots reference async descriptors; coroutine and future kinds need separate decoders. const synchronous = (methodFlags & 0x40) === 0 && (methodFlags & 15) <= 3; - const implementation = read.relative(entry + 4n); - const resolved = synchronous && read.executable(implementation); + const implementationValue = read.relative( + entry + 4n, + "methodImplementation", + ); + const implementation = + implementationValue.kind === "target" + ? implementationValue.target + : null; + const implementationReason = + implementationValue.kind === "target" + ? null + : unresolvedSwiftRelativeReason( + implementationValue, + "methodImplementation", + ); + const resolved = + implementation !== null && + synchronous && + read.executable(implementation); result.swift_dispatch_slots.push({ owner, table_kind: "class_vtable", @@ -108,7 +141,9 @@ export const decodeSwiftClassVtables = (input: { : { status: "partial", reason: - "Async, coroutine, external or unmapped vtable implementation is unresolved", + implementation === null + ? implementationReason + : "Async, coroutine, external or unmapped vtable implementation is unresolved", }, evidence: read.evidence( entry, @@ -121,7 +156,9 @@ export const decodeSwiftClassVtables = (input: { issue( "swift_class_vtable_descriptors", "vtable_implementation_unresolved", - "Async, coroutine, external or unmapped vtable implementation is unresolved", + implementation === null + ? implementationReason! + : "Async, coroutine, external or unmapped vtable implementation is unresolved", `0x${entry.toString(16)}`, ), ); From 335c042d166b2c8b2c0600935cf77607ae11edbc Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:26:43 +0800 Subject: [PATCH 35/35] docs(testing): reference the configured duplication limit (#5758) --- docs/testing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 51f3f65ce0..5ff3c4712e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -78,8 +78,9 @@ without the artifacts lock, so it can record hashes while `check:pr` or `check:test` already holds that lock. The CI jobs still run the locked `check:ci:fast` and `check:ci:deep` commands. Use `npm run lint:details` or `npm run jscpd:details` for all console diagnostics. -Both duplicate commands scan production and test code with the same 3% limit; -the summary commands preserve checker failures and restore reports from cached +Both duplicate commands scan production and test code using the limit in +[`.jscpd.json`](https://github.com/morluto/rea/blob/main/.jscpd.json). +The summary commands preserve checker failures and restore reports from cached static tasks. Both lint caches include the website, documentation configuration, and experiment code scanned by the checker; a regression compares the actual lint inventory with Turbo's resolved inputs.