From 23de10bb2849cdce8ddbe70504aa7cc57de75a98 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:43:04 +0800 Subject: [PATCH] fix: streamline tests and preserve Darwin process ownership (#8275) * test: strengthen suite guards and regression oracles * test: share JSON client registration contract * test: reuse fixtures and reduce repeated setup * fix(process): preserve ambiguous Darwin token ownership * test(process): bound child readiness waits --- bridge/process/ProcessRunTokenReader.swift | 47 +++- scripts/run-python-tests.py | 39 ++- .../ProcessOwnership.finalSweep.test.ts | 90 +++++-- src/process/ProcessOwnership.ts | 12 + .../ProcessOwnership.validation.test.ts | 36 +++ .../exportedFunctionBindingsCli.test.ts | 42 ++- .../javascriptReturnedReferences.test.ts | 69 ++--- tests/acceptance/gpuCodeObjectsCli.test.ts | 4 +- tests/acceptance/pythonPycCli.test.ts | 16 +- .../setup/hermesAliasedTableSetup.test.ts | 42 ++- .../setup/hermesAnchoredEntrySetup.test.ts | 246 +++++++++++++----- tests/acceptance/sqliteDatabaseCli.test.ts | 23 +- tests/boundary/browser/sazWebCapture.test.ts | 34 ++- tests/boundary/cli/analysisViewCli.test.ts | 36 ++- .../cli/clientProfileLifecycle.test.ts | 11 +- tests/boundary/cli/clientProfiles.test.ts | 11 +- .../boundary/filesystem/ciTestReports.test.ts | 59 +++++ .../javascriptArtifactReconstruction.test.ts | 225 ++++++++-------- .../qoderClientConfiguration.test.ts | 117 +-------- .../workbuddyClientConfiguration.test.ts | 115 +------- tests/boundary/mcp/deferredRuntime.test.ts | 35 ++- tests/boundary/mcp/largeToolResult.test.ts | 80 +++--- tests/boundary/mcp/mcpDoctor.test.ts | 89 ++++--- .../mcp/optionalProviderIsolation.test.ts | 14 +- tests/boundary/pcap/pcapProvider.test.ts | 60 ----- .../pcap/pcapProviderCrossPlatform.test.ts | 10 +- .../darwinProcessRunTokenReader.test.ts | 128 ++++++++- .../contracts/mcpResponseBudget.test.ts | 42 +-- tests/fixtures/javascriptArtifactAnalysis.ts | 70 +++-- tests/fixtures/javascriptReturnShapes.ts | 32 ++- tests/fixtures/localToolsMcp.ts | 7 +- .../fixtures/processRunTokenParserProbe.swift | 101 +++++-- .../support/jsonClientRegistrationContract.ts | 126 +++++++++ 33 files changed, 1297 insertions(+), 771 deletions(-) create mode 100644 tests/support/jsonClientRegistrationContract.ts diff --git a/bridge/process/ProcessRunTokenReader.swift b/bridge/process/ProcessRunTokenReader.swift index 9f2456fe49..d5152ab50c 100644 --- a/bridge/process/ProcessRunTokenReader.swift +++ b/bridge/process/ProcessRunTokenReader.swift @@ -161,8 +161,20 @@ private func unavailableIdentity(pid: Int32, reason: String) -> ProcessIdentityO ) } +enum ProcargsMalformedStage: String, CaseIterable { + case header + case argumentCount = "argument_count" + case executablePath = "executable_path" + case pointerSize = "pointer_size" + case pathPadding = "path_padding" + case arguments + case terminator + case trailingEntries = "trailing_entries" +} + enum RunTokenReadError: Error { case invalidBuffer + case invalidBufferAt(ProcargsMalformedStage) case duplicateToken case invalidTokenEncoding case environmentUnavailable @@ -244,6 +256,8 @@ func processRunTokenObservation( return unavailable(pid: pid, reason: "apple_vector_unavailable") case .invalidBuffer: return unavailable(pid: pid, reason: "malformed_procargs") + case .invalidBufferAt(let stage): + return unavailable(pid: pid, reason: "malformed_procargs_\(stage.rawValue)") } } } @@ -302,27 +316,30 @@ func parseProcessEnvironment( in bytes: [UInt8], pointerSize: Int = MemoryLayout.size ) throws -> ParsedRunToken { - guard bytes.count >= MemoryLayout.size else { throw RunTokenReadError.invalidBuffer } + guard bytes.count >= MemoryLayout.size else { throw RunTokenReadError.invalidBufferAt(.header) } let argc = bytes.withUnsafeBytes { $0.loadUnaligned(as: Int32.self).littleEndian } - guard argc >= 0, Int(argc) <= bytes.count else { throw RunTokenReadError.invalidBuffer } + guard argc >= 0, Int(argc) <= bytes.count else { throw RunTokenReadError.invalidBufferAt(.argumentCount) } var offset = MemoryLayout.size guard let executablePath = readString(in: bytes, offset: &offset) else { - throw RunTokenReadError.invalidBuffer + throw RunTokenReadError.invalidBufferAt(.executablePath) } - guard pointerSize == 4 || pointerSize == 8 else { throw RunTokenReadError.invalidBuffer } + guard pointerSize == 4 || pointerSize == 8 else { throw RunTokenReadError.invalidBufferAt(.pointerSize) } let pathRecordSize = Array("executable_path=".utf8).count + executablePath.count + 1 let padding = (pointerSize - pathRecordSize % pointerSize) % pointerSize guard offset + padding <= bytes.count, bytes[offset..<(offset + padding)].allSatisfy({ $0 == 0 }) else { - throw RunTokenReadError.invalidBuffer + throw RunTokenReadError.invalidBufferAt(.pathPadding) } offset += padding + let tokenPrefix = Array("REA_PROCESS_RUN_ID=".utf8) + var tokenInArgument = false for _ in 0.. 2 and parts[0] == "bridge": - if parts[1] in REQUIRED_BRIDGE_FAMILIES: - self.completed.add(parts[1]) + self.completed.add(test.id()) def startTest(self, test): self.active_test = test @@ -65,11 +62,16 @@ def run_bridge(modules): # The script directory is sys.path[0]; imports must resolve from the checkout. sys.path.insert(0, str(root)) suite = unittest.defaultTestLoader.loadTestsFromNames(modules) - result = unittest.TextTestRunner(verbosity=2, resultclass=BridgeResult).run(suite) + result = unittest.TextTestRunner(verbosity=2, resultclass=ExecutedResult).run(suite) + completed_families = { + parts[1] + for name in result.completed + if len(parts := name.split(".")) > 2 and parts[0] == "bridge" + } missing = [ family for family in REQUIRED_BRIDGE_FAMILIES - if family not in result.completed + if family not in completed_families ] if missing: fail("no_executed_bridge_tests", families=missing) @@ -77,6 +79,19 @@ def run_bridge(modules): raise SystemExit(1) +def run_analysis_efficiency(): + # Match the checkout-root import path of `python -m unittest discover`. + sys.path.insert(0, str(root)) + suite = unittest.defaultTestLoader.discover( + str(root / "tests/support/analysis-efficiency"), pattern="test_*.py" + ) + result = unittest.TextTestRunner(verbosity=2, resultclass=ExecutedResult).run(suite) + if not result.wasSuccessful(): + raise SystemExit(1) + if not result.completed: + fail("no_executed_analysis_efficiency_tests") + + bridge_root = root / "bridge" bridge_tests = sorted( path for path in bridge_root.rglob("test_*.py") if path.is_file() @@ -95,6 +110,9 @@ modules = [ if sys.argv[1:] == ["--bridge"]: run_bridge(modules) raise SystemExit(0) +if sys.argv[1:] == ["--analysis-efficiency"]: + run_analysis_efficiency() + raise SystemExit(0) if sys.argv[1:]: fail("invalid_python_test_arguments") bridge_result = subprocess.run( @@ -103,10 +121,7 @@ bridge_result = subprocess.run( if bridge_result.returncode: sys.exit(bridge_result.returncode) result = subprocess.run( - [ - sys.executable, "-m", "unittest", "discover", - "-s", "tests/support/analysis-efficiency", "-p", "test_*.py", "-v", - ], + [sys.executable, __file__, "--analysis-efficiency"], cwd=root, check=False, ) diff --git a/src/process/ProcessOwnership.finalSweep.test.ts b/src/process/ProcessOwnership.finalSweep.test.ts index 2f3d5036a1..ab353eef56 100644 --- a/src/process/ProcessOwnership.finalSweep.test.ts +++ b/src/process/ProcessOwnership.finalSweep.test.ts @@ -167,49 +167,83 @@ it("finishes the public cleanup sweep when the application timer is unavailable" } }); -it("keeps an unreadable live group member fail-closed when the application timer is unavailable", async () => { - const fixture = makeHost(() => undefined); - let unreadableReads = 0; +it.each([ + "EACCES: fixture permission denied", + "ambiguous_environment_boundary", +])( + "keeps an unreadable live group member fail-closed when the application timer is unavailable: %s", + async (diagnostic) => { + const fixture = makeHost(() => undefined); + let unreadableReads = 0; + const host: ProcessOwnershipHost = { + ...fixture.host, + runTokens: async (entries) => + new Map( + entries.map(({ pid }) => { + if (pid === 101) { + unreadableReads += 1; + return [ + pid, + { + state: "unavailable", + reason: diagnostic, + }, + ] as const; + } + return [pid, { state: "readable", runId: "sweep-run" }] as const; + }), + ), + }; + const applicationTimer = vi + .spyOn(globalThis, "setTimeout") + .mockImplementation(() => { + throw new Error("application timer is unavailable"); + }); + + try { + await expect( + cleanupOwnedProcessGroup(ownership(), host), + ).resolves.toMatchObject({ + cleaned: false, + reason: expect.stringContaining( + "process ownership token could not be read", + ), + }); + expect(unreadableReads).toBeGreaterThan(1); + expect(fixture.signalGroups).toEqual([]); + expect(applicationTimer).not.toHaveBeenCalled(); + } finally { + applicationTimer.mockRestore(); + } + }, +); + +it("retries an ambiguous owned member before signaling its group", async () => { + const fixture = makeHost(() => fixture.replaceRows([])); + let memberReads = 0; const host: ProcessOwnershipHost = { ...fixture.host, runTokens: async (entries) => new Map( entries.map(({ pid }) => { - if (pid === 101) { - unreadableReads += 1; + if (pid === 101 && memberReads++ === 0) return [ pid, { state: "unavailable", - reason: "EACCES: fixture permission denied", + reason: "ambiguous_environment_boundary", }, ] as const; - } return [pid, { state: "readable", runId: "sweep-run" }] as const; }), ), }; - const applicationTimer = vi - .spyOn(globalThis, "setTimeout") - .mockImplementation(() => { - throw new Error("application timer is unavailable"); - }); - - try { - await expect( - cleanupOwnedProcessGroup(ownership(), host), - ).resolves.toMatchObject({ - cleaned: false, - reason: expect.stringContaining( - "process ownership token could not be read", - ), - }); - expect(unreadableReads).toBeGreaterThan(1); - expect(fixture.signalGroups).toEqual([]); - expect(applicationTimer).not.toHaveBeenCalled(); - } finally { - applicationTimer.mockRestore(); - } + await expect(cleanupOwnedProcessGroup(ownership(), host)).resolves.toEqual({ + cleaned: true, + signaled: true, + }); + expect(memberReads).toBeGreaterThan(1); + expect(fixture.signalGroups).toEqual([100]); }); it("preserves a moved same-generation helper as unresolved through the final deadline", async () => { diff --git a/src/process/ProcessOwnership.ts b/src/process/ProcessOwnership.ts index cc9d838adc..eb47356f7a 100644 --- a/src/process/ProcessOwnership.ts +++ b/src/process/ProcessOwnership.ts @@ -1278,6 +1278,18 @@ const sanitizedTokenReadFailure = (diagnostic: string | undefined): string => { if (diagnostic === "apple_vector_unavailable") return diagnostic; if (diagnostic === "ambiguous_environment_boundary") return diagnostic; if (diagnostic === "malformed_procargs") return diagnostic; + // The native parser reports only fixed stage names, never process data. + if ( + diagnostic === "malformed_procargs_header" || + diagnostic === "malformed_procargs_argument_count" || + diagnostic === "malformed_procargs_executable_path" || + diagnostic === "malformed_procargs_pointer_size" || + diagnostic === "malformed_procargs_path_padding" || + diagnostic === "malformed_procargs_arguments" || + diagnostic === "malformed_procargs_terminator" || + diagnostic === "malformed_procargs_trailing_entries" + ) + return diagnostic; if (diagnostic === "process_identity_changed_during_token_read") return diagnostic; if (diagnostic === "process_not_in_snapshot") return diagnostic; diff --git a/src/process/ProcessOwnership.validation.test.ts b/src/process/ProcessOwnership.validation.test.ts index 7466c5fe84..9e9c7d6a3c 100644 --- a/src/process/ProcessOwnership.validation.test.ts +++ b/src/process/ProcessOwnership.validation.test.ts @@ -160,6 +160,42 @@ describe("owned process-group cleanup validation: ownership and lineage", () => }); describe("owned process-group cleanup liveness rechecks", () => { + it("reports fixed malformed procargs stages and sanitizes the cleanup summary", async () => { + const reasons = [ + "malformed_procargs_header", + "malformed_procargs_argument_count", + "malformed_procargs_executable_path", + "malformed_procargs_pointer_size", + "malformed_procargs_path_padding", + "malformed_procargs_arguments", + "malformed_procargs_terminator", + "malformed_procargs_trailing_entries", + "malformed_procargs_private-data", + ]; + const members = reasons.map((_, index) => ({ + pid: 100 + index, + parentPid: 100, + processGroupId: 100, + state: "S", + command: "fixture", + })); + const signalGroup = vi.fn(); + const adapter: ProcessOwnershipHost = { + listProcesses: async () => members, + environment: async (pid) => { + throw new Error(reasons[pid - 100]); + }, + signalGroup, + }; + const result = await cleanupOwnedProcessGroup(ownership, adapter); + expect(result.cleaned).toBe(false); + if (result.cleaned) throw new Error("expected cleanup to retain ownership"); + for (const reason of reasons.slice(0, -1)) + expect(result.reason).toContain(reason); + expect(result.reason).toContain("108=other_unavailable"); + expect(result.reason).not.toContain("private-data"); + expect(signalGroup).not.toHaveBeenCalled(); + }); it("rechecks all unreadable members after the token phase in one fresh snapshot", async () => { const members = [100, 101, 102, 103].map((pid) => ({ pid, diff --git a/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts b/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts index 9814118039..020f147e99 100644 --- a/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts +++ b/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts @@ -1115,11 +1115,15 @@ const expectRuntimeExports = async ( expected: (count: number) => unknown, ): Promise => { // Execute only source-owned fixtures as an independent runtime oracle. - for (const [path, exportName, count] of [ + const entries = [ [fixture.leftPath, fixture.input.left_export_name, 1], [fixture.rightPath, fixture.input.right_export_name, 2], - ] as const) - expect(await runtimeExport(path, exportName)).toEqual(expected(count)); + ] as const; + const values = await runtimeExports( + entries.map(([path, exportName]) => [path, exportName]), + ); + for (const [index, [, , count]] of entries.entries()) + expect(values[index]).toEqual(expected(count)); }; const expectExportedCallableBinding = ( @@ -1333,20 +1337,30 @@ const getMcpClient = async (): Promise => { return mcpConnection.client; }; -const runtimeExport = async ( - path: string, - exportName: string, -): Promise => { +const runtimeExports = async ( + entries: readonly (readonly [path: string, exportName: string])[], +): Promise => { const { stdout } = await execute(process.execPath, [ "--input-type=module", "-e", `import { pathToFileURL } from 'node:url'; - const module = await import(pathToFileURL(process.argv[1]).href); - const value = module[process.argv[2]]; - console.log(JSON.stringify({ value: typeof value === 'function' ? value() : value }));`, - path, - exportName, + const results = []; + for (const [path, exportName] of JSON.parse(process.argv[1])) { + const module = await import(pathToFileURL(path).href); + const value = module[exportName]; + results.push({ value: typeof value === 'function' ? value() : value }); + } + console.log(JSON.stringify(results));`, + JSON.stringify(entries), ]); - return z.object({ value: z.unknown().optional() }).parse(JSON.parse(stdout)) - .value; + return z + .array(z.object({ value: z.unknown().optional() })) + .length(entries.length) + .parse(JSON.parse(stdout)) + .map(({ value }) => value); }; + +const runtimeExport = async ( + path: string, + exportName: string, +): Promise => (await runtimeExports([[path, exportName]]))[0]; diff --git a/tests/acceptance/applications/javascriptReturnedReferences.test.ts b/tests/acceptance/applications/javascriptReturnedReferences.test.ts index d10ccf47ba..8ad3a8da1a 100644 --- a/tests/acceptance/applications/javascriptReturnedReferences.test.ts +++ b/tests/acceptance/applications/javascriptReturnedReferences.test.ts @@ -2,6 +2,7 @@ import { expect } from "vitest"; import { verifyJavaScriptReturnShapes, + verifyJavaScriptReturnShapesThroughCli, type JavaScriptReturnFields, } from "../../fixtures/javascriptReturnShapes.js"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -60,6 +61,22 @@ const writes = { (_, index) => `m${index}() { return shared; }`, ).join(", ")} }; box.m0().x = 2; box.m63().other = 3;`, }; +const wideReceiverSource = (width: number): string => { + const object = Array.from( + { length: width }, + (_, index) => `p${index}:0`, + ).join(","); + const array = "0,".repeat(width); + return [ + ["wideObject", `{${object},last:shared}`], + ["wideArray", `[${array}shared]`], + ] + .map( + ([name, value]) => + `export function ${name}(){const shared={x:1};const box={wide(){return ${value}},noop(){}};box.noop();return {x:shared.x,keep:7};}`, + ) + .join("\n"); +}; const source = [ ...Object.entries(writes).map( ([name, body]) => @@ -93,8 +110,20 @@ const source = [ "export function unrelated(){ const shared = {x: 1}; const other = {x: 9}; const get = () => shared; get().x = 2; return other.x; }", "export function receiverDistinctScopes(){ const shared = {x: 1}; const box = {outer(){ return shared; }, inner(){ const shared = {x: 9}; return shared; }, noop(){}}; box.noop(); return {outer: shared.x, inner: box.inner().x}; }", "export function receiverDistinctPaths(){ const parent = {left: {x: 1}, right: {x: 9}, keep: 7}; const box = {left(){ return parent.left; }, right(){ return parent.right; }, noop(){}}; box.noop(); return {left: parent.left.x, right: parent.right.x, keep: parent.keep}; }", + wideReceiverSource(64), ].join("\n"); +const assertWideReturns = (fields: JavaScriptReturnFields): void => { + for (const name of ["wideObject", "wideArray"]) { + expect(fields(name)).toContainEqual( + expect.objectContaining({ path: "/x", state: "unknown" }), + ); + expect(fields(name)).toContainEqual( + expect.objectContaining({ path: "/keep", state: "literal", value: 7 }), + ); + } +}; + const assertReturns = (fields: JavaScriptReturnFields): void => { for (const name of Object.keys(writes)) { const field = fields(name).find(({ path }) => path === ""); @@ -176,7 +205,11 @@ const assertReturns = (fields: JavaScriptReturnFields): void => { cliTest( "preserves mutation uncertainty for objects returned by local calls", - ({ cli }) => verifyJavaScriptReturnShapes(cli, source, assertReturns), + ({ cli }) => + verifyJavaScriptReturnShapes(cli, source, (fields) => { + assertReturns(fields); + assertWideReturns(fields); + }), 120_000, ); @@ -185,35 +218,11 @@ cliTest( ({ cli }) => { // Exceed the argument limit with the worker's 2048 KiB stack as well as // Node's default stack, so a spread into push still fails this regression. - const width = 300_000; - const object = Array.from( - { length: width }, - (_, index) => `p${index}:0`, - ).join(","); - const array = "0,".repeat(width); - const wideSource = [ - ["wideObject", `{${object},last:shared}`], - ["wideArray", `[${array}shared]`], - ] - .map( - ([name, value]) => - `export function ${name}(){const shared={x:1};const box={wide(){return ${value}},noop(){}};box.noop();return {x:shared.x,keep:7};}`, - ) - .join("\n"); - return verifyJavaScriptReturnShapes(cli, wideSource, (fields) => { - for (const name of ["wideObject", "wideArray"]) { - expect(fields(name)).toContainEqual( - expect.objectContaining({ path: "/x", state: "unknown" }), - ); - expect(fields(name)).toContainEqual( - expect.objectContaining({ - path: "/keep", - state: "literal", - value: 7, - }), - ); - } - }); + return verifyJavaScriptReturnShapesThroughCli( + cli, + wideReceiverSource(300_000), + assertWideReturns, + ); }, 120_000, ); diff --git a/tests/acceptance/gpuCodeObjectsCli.test.ts b/tests/acceptance/gpuCodeObjectsCli.test.ts index 5edcf3b942..c48600d867 100644 --- a/tests/acceptance/gpuCodeObjectsCli.test.ts +++ b/tests/acceptance/gpuCodeObjectsCli.test.ts @@ -38,7 +38,7 @@ cliTest( ] as const) { const path = join(root, name); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -817,7 +817,7 @@ cliTest( "attributes malformed selector JSON to its actual CLI option", async ({ cli }) => { for (const name of ["object_ids", "targets", "view"]) { - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", "/not-opened-for-invalid-selectors", diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index 0105487421..4f3b56b39a 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -278,7 +278,7 @@ cliTest( header[2] = 0x0d; header[3] = 0x0a; await writeFile(path, Buffer.concat([header, marshal])); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -295,7 +295,7 @@ cliTest( for (const magic of [3392, 3394]) { const path = join(root, `${magic}.pyc`); await writeFile(path, Buffer.concat([pythonPycHeader(magic), marshal])); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -354,7 +354,7 @@ cliTest( path, Buffer.concat([pythonPycHeader(magic), marshalCode(layout, fields)]), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -413,7 +413,7 @@ cliTest( path, Buffer.concat([pythonPycHeader(magic), marshalCode(layout, fields)]), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -502,7 +502,7 @@ cliTest( }), ]), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -552,7 +552,7 @@ cliTest( marshalCode(testCase.layout, { consts: [frozenSet, dict] }), ]), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -633,7 +633,7 @@ cliTest( marshalCode(testCase.layout, { consts: [frozenSet, dict] }), ]), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -664,7 +664,7 @@ cliTest( }), ]), ); - const rawResult = await cli.run({ + const rawResult = await runFileCli(cli, { arguments: ["inspect-python-pyc", rawPath, "--json"], }); expect(rawResult.exitCode, rawResult.stderr).toBe(0); diff --git a/tests/acceptance/setup/hermesAliasedTableSetup.test.ts b/tests/acceptance/setup/hermesAliasedTableSetup.test.ts index 223d8465e5..6b8a913be2 100644 --- a/tests/acceptance/setup/hermesAliasedTableSetup.test.ts +++ b/tests/acceptance/setup/hermesAliasedTableSetup.test.ts @@ -1,8 +1,9 @@ import { access, mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { parse, stringify } from "yaml"; import { expect } from "vitest"; +import { currentClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -238,16 +239,15 @@ cliTest( }, timeoutMs: 20_000, }); - const setup = await run([ - "setup", - "--client", - "hermes", - "--skill=false", - "--yes", - "--json", - ]); - expect(setup.exitCode).toBe(0); - const entry = parse(await readFile(config, "utf8")).mcp_servers.rea; + const [command, ...args] = currentClientRegistrationCommand({ + entrypointPath: resolve("scripts/rea.mjs"), + }); + const entry = { + command, + args, + env: { HOPPER_LAUNCHER_PATH: process.execPath }, + enabled: true, + }; const defaults = { rea: entry, other: { command: "other-server" } }; const original = "server_defaults: &servers\n" + @@ -258,9 +258,23 @@ cliTest( .join("") + "mcp_servers: *servers\nobserver: *servers\n"; await writeFile(config, original); - expect( - (await run(["doctor", "--client", "hermes", "--json"])).exitCode, - ).toBe(0); + await expect(access(`${config}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + const doctor = await run(["doctor", "--client", "hermes", "--json"]); + expect(doctor.exitCode, doctor.stdout).toBe(0); + expect(doctor.json).toMatchObject({ + healthy: true, + identity: { + registrations: expect.arrayContaining([ + expect.objectContaining({ + client: "hermes", + config_path: config, + state: "aligned", + }), + ]), + }, + }); const removed = await run(["uninstall", "--json"]); expect(removed.exitCode).toBe(0); const remaining = parse(await readFile(config, "utf8")); diff --git a/tests/acceptance/setup/hermesAnchoredEntrySetup.test.ts b/tests/acceptance/setup/hermesAnchoredEntrySetup.test.ts index 22948632ae..92c65b2276 100644 --- a/tests/acceptance/setup/hermesAnchoredEntrySetup.test.ts +++ b/tests/acceptance/setup/hermesAnchoredEntrySetup.test.ts @@ -1,10 +1,160 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { parse } from "yaml"; -import { expect } from "vitest"; +import { expect, onTestFinished } from "vitest"; + +import { currentClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; +import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; +import { supportedClients } from "../../../src/application/SupportedClients.js"; +import { systemUninstallHost } from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; + +const cliLifecycle = (cli: TestCli, home: string, profile: string) => { + const client = supportedClients(home, process.platform, { + HERMES_HOME: profile, + }).find(({ name }) => name === "hermes"); + if (client?.configPath !== join(profile, "config.yaml")) + throw new Error("Hermes profile did not resolve to the test config"); + const run = (arguments_: readonly string[]) => + cli.run({ + arguments: arguments_, + cwd: home, + environment: { HOME: home, USERPROFILE: home, HERMES_HOME: profile }, + timeoutMs: 20_000, + }); + const setup = [ + "setup", + "--client", + "hermes", + "--skill=false", + "--yes", + "--json", + ]; + const remove = async () => { + const result = await run(["uninstall", "--json"]); + expect(result.exitCode, JSON.stringify(result)).toBe(0); + }; + return { + configure: async () => { + const result = await run(setup); + expect(result.exitCode, JSON.stringify(result)).toBe(0); + expect(result.json).toMatchObject({ status: "ready" }); + }, + verifyAndRepeat: async () => { + const doctor = await run(["doctor", "--client", "hermes", "--json"]); + expect(doctor.exitCode, JSON.stringify(doctor)).toBe(0); + expect(doctor.json).toMatchObject({ + healthy: true, + identity: { registrations: [{ client: "hermes", state: "aligned" }] }, + }); + const repeated = await run(setup); + expect(repeated.exitCode, JSON.stringify(repeated)).toBe(0); + expect(repeated.json).toMatchObject({ + plannedActions: [], + appliedActions: [], + }); + }, + remove, + removeAgain: remove, + }; +}; + +const configurationLifecycle = ( + home: string, + profile: string, + config: string, + command: readonly string[], +) => { + const environment = { HERMES_HOME: profile }; + const client = supportedClients(home, process.platform, environment).find( + ({ name }) => name === "hermes", + ); + if (client === undefined || client.configPath !== config) + throw new Error("Hermes profile did not resolve to the test config"); + const uninstallHost = systemUninstallHost( + home, + undefined, + environment, + process.platform, + ); + onTestFinished(async () => { + expect(await uninstallHost.close?.()).toBeUndefined(); + }); + return { + configure: async () => { + expect( + await configureClientConfiguration(client, {}, command), + ).toMatchObject({ status: "configured" }); + }, + verifyAndRepeat: async () => { + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment, + selectedClientIds: ["hermes"], + }) + ).find(({ client: name }) => name === "hermes"), + ).toMatchObject({ state: "aligned", config_path: config }); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "unchanged", + }); + }, + remove: async () => { + expect((await uninstallHost.removeClient(client)).status).toBe("removed"); + }, + removeAgain: async () => { + expect((await uninstallHost.removeClient(client)).status).toBe("skipped"); + }, + }; +}; + +type AnchorStyle = "entry" | "child" | "shadowed" | "key" | "external"; + +const anchoredConfiguration = ( + version: "1.1" | "1.2", + style: AnchorStyle, + server: { + command: string; + args: string[]; + env: { KEEP: string }; + enabled: boolean; + }, +) => { + const directive = version === "1.2" ? "%YAML 1.2\n---\n" : ""; + const hasDefaults = style === "shadowed" || style === "external"; + const entry = + ` command: ${JSON.stringify(server.command)}\n` + + ` args: ${JSON.stringify(server.args)}\n` + + " enabled: false\n" + + (hasDefaults + ? " env: *settings # Keep nested reference comment\n" + : ` env: ${style === "child" ? "&settings " : ""}{KEEP: 'off'}\n`); + const original = + directive + + "# Keep this user setting\nmodel: custom\n" + + (hasDefaults ? "defaults: &settings {KEEP: 'off'}\n" : "") + + `mcp_servers:\n ${style === "key" ? "&server_name " : ""}rea:${style === "child" || style === "key" ? "" : " &server"}${style === "entry" ? " # Keep anchor definition comment" : ""}\n` + + entry + + (style === "child" + ? " other:\n command: other-server\n env: *settings # Shared environment\n" + : style === "key" + ? " other: {command: other-server}\nobserver: *server_name # Registration name\n" + : " other: *server # Independent server\n") + + (style === "shadowed" + ? "later: &settings {KEEP: later}\nobserver: *server\n" + : ""); + const other = + style === "child" + ? { command: "other-server", env: { KEEP: "off" } } + : style === "key" + ? { command: "other-server" } + : server; + return { original, other, hasDefaults }; +}; for (const version of ["1.1", "1.2"] as const) { for (const style of [ @@ -14,73 +164,39 @@ for (const version of ["1.1", "1.2"] as const) { "key", "external", ] as const) { + const publicCli = style === "entry" || style === "shadowed"; cliTest( - `Hermes ${version} preserves ${style} anchor consumers through setup and uninstall`, + `Hermes ${version} preserves ${style} anchor consumers through ${publicCli ? "CLI" : "configuration boundary"} setup and uninstall`, async ({ cli }) => { + // Exercise both YAML versions through the CLI, including the shadowed + // anchor case; the other styles retain the full registration lifecycle + // at the production configuration and uninstall boundary. const home = await createTestTempDirectory( "rea-hermes-anchored-entry-", ); const profile = join(home, "hermes"); const config = join(profile, "config.yaml"); + const command = publicCli + ? [process.execPath, resolve("scripts/rea.mjs"), "mcp"] + : currentClientRegistrationCommand({ platform: process.platform }); + if (command[0] === undefined) throw new Error("Missing REA entrypoint"); const server = { - command: process.execPath, - args: [resolve("scripts/rea.mjs"), "mcp"], + command: command[0], + args: command.slice(1), env: { KEEP: "off" }, enabled: false, }; - const directive = version === "1.2" ? "%YAML 1.2\n---\n" : ""; - const hasDefaults = style === "shadowed" || style === "external"; - const entry = - ` command: ${JSON.stringify(server.command)}\n` + - ` args: ${JSON.stringify(server.args)}\n` + - " enabled: false\n" + - (hasDefaults - ? " env: *settings # Keep nested reference comment\n" - : ` env: ${style === "child" ? "&settings " : ""}{KEEP: 'off'}\n`); - const original = - directive + - "# Keep this user setting\nmodel: custom\n" + - (hasDefaults ? "defaults: &settings {KEEP: 'off'}\n" : "") + - `mcp_servers:\n ${style === "key" ? "&server_name " : ""}rea:${style === "child" || style === "key" ? "" : " &server"}${style === "entry" ? " # Keep anchor definition comment" : ""}\n` + - entry + - (style === "child" - ? " other:\n command: other-server\n env: *settings # Shared environment\n" - : style === "key" - ? " other: {command: other-server}\nobserver: *server_name # Registration name\n" - : " other: *server # Independent server\n") + - (style === "shadowed" - ? "later: &settings {KEEP: later}\nobserver: *server\n" - : ""); - const other = - style === "child" - ? { command: "other-server", env: { KEEP: "off" } } - : style === "key" - ? { command: "other-server" } - : server; + const { original, other, hasDefaults } = anchoredConfiguration( + version, + style, + server, + ); await mkdir(profile); await writeFile(config, original); - const run = (arguments_: readonly string[]) => - cli.run({ - arguments: arguments_, - cwd: home, - environment: { - HOME: home, - USERPROFILE: home, - HERMES_HOME: profile, - }, - timeoutMs: 20_000, - }); - const setup = [ - "setup", - "--client", - "hermes", - "--skill=false", - "--yes", - "--json", - ]; - const applied = await run(setup); - expect(applied.exitCode, JSON.stringify(applied)).toBe(0); - expect(applied.json).toMatchObject({ status: "ready" }); + const lifecycle = publicCli + ? cliLifecycle(cli, home, profile) + : configurationLifecycle(home, profile, config, command); + await lifecycle.configure(); const configured = await readFile(config, "utf8"); const configuredValues = parse(configured, { version }); expect(configuredValues).toMatchObject({ @@ -108,21 +224,9 @@ for (const version of ["1.1", "1.2"] as const) { if (hasDefaults) expect(configured).toContain("# Keep nested reference comment"); expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); - const doctor = await run(["doctor", "--client", "hermes", "--json"]); - expect(doctor.exitCode, JSON.stringify(doctor)).toBe(0); - expect(doctor.json).toMatchObject({ - healthy: true, - identity: { registrations: [{ client: "hermes", state: "aligned" }] }, - }); - const repeated = await run(setup); - expect(repeated.exitCode, JSON.stringify(repeated)).toBe(0); - expect(repeated.json).toMatchObject({ - plannedActions: [], - appliedActions: [], - }); + await lifecycle.verifyAndRepeat(); expect(await readFile(config, "utf8")).toBe(configured); - const removed = await run(["uninstall", "--json"]); - expect(removed.exitCode, JSON.stringify(removed)).toBe(0); + await lifecycle.remove(); const removedValues = parse(await readFile(config, "utf8"), { version, }); @@ -148,7 +252,7 @@ for (const version of ["1.1", "1.2"] as const) { expect(remaining).toContain("# Keep anchor definition comment"); if (hasDefaults) expect(remaining).toContain("# Keep nested reference comment"); - expect((await run(["uninstall", "--json"])).exitCode).toBe(0); + await lifecycle.removeAgain(); expect(await readFile(config, "utf8")).toBe(remaining); }, ); diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index 206c781da2..49f268f3e3 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -10,7 +10,7 @@ import { } from "node:fs/promises"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; -import { expect, onTestFinished } from "vitest"; +import { afterAll, expect, onTestFinished } from "vitest"; import { z } from "zod"; import { expandProviderCapabilities, @@ -30,11 +30,24 @@ import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { expectCliMcpEvidenceParity } from "../fixtures/cliMcpEvidenceParity.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; +import { + cliTest, + createSharedMcpSession, + runFileCli, +} from "../support/cli/cliFixture.js"; const digest = (bytes: Buffer): string => createHash("sha256").update(bytes).digest("hex"); +// These stateless parity cases use the same default admission settings. Keep +// their server file-owned, while lifecycle and configuration cases use fresh +// connections with test-owned cleanup. +const parityMcp = createSharedMcpSession(async (registerCleanup, isClosed) => { + if (isClosed()) throw new Error("SQLite parity MCP session is closed"); + return (await connectLocalToolsMcp({}, registerCleanup)).call; +}); +afterAll(() => parityMcp.close()); + cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "unions SQLite cell slices with numeric cell paths", async ({ cli }) => { @@ -719,7 +732,7 @@ cliTest }, }, }); - const { call } = await connectLocalToolsMcp(); + const call = await parityMcp.session.call(); const response = await call("inspect_sqlite_database", { path, table: 'odd " table', @@ -1197,7 +1210,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( writer.close(); } const original = await readFile(path); - const { call } = await connectLocalToolsMcp(); + const call = await parityMcp.session.call(); for (const schema_scope of [ "inventory", "selected_table", @@ -1413,7 +1426,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "folds ASCII trigger spelling without merging non-ASCII UTF-16 table names", async ({ cli }) => { const root = await createTestTempDirectory("rea-sqlite-utf16-identity-"); - const { call } = await connectLocalToolsMcp(); + const call = await parityMcp.session.call(); for (const encoding of ["UTF-16le", "UTF-16be"]) { const path = join(root, `${encoding}.db`); const producer = new DatabaseSync(path); diff --git a/tests/boundary/browser/sazWebCapture.test.ts b/tests/boundary/browser/sazWebCapture.test.ts index f29235c0a1..a6339fe8c5 100644 --- a/tests/boundary/browser/sazWebCapture.test.ts +++ b/tests/boundary/browser/sazWebCapture.test.ts @@ -7,7 +7,13 @@ import { fileURLToPath } from "node:url"; import { gzipSync } from "node:zlib"; import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect, it as runtimeTest } from "vitest"; +import { + afterAll, + expect, + it as runtimeTest, + onTestFinished, + vi, +} from "vitest"; import { z } from "zod"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { @@ -15,6 +21,7 @@ import { webNetworkCaptureSchema, } from "../../../src/domain/webNetworkCapture.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { protobufFrame } from "../../fixtures/protobufCapture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -29,6 +36,12 @@ const hasMemoryBound = const it = runtimeTest.skipIf(!hasMemoryBound); +// Share schema compilation, not sessions or their retained Evidence. +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); +const connectSazMcp = () => + connectLocalToolsMcp({}, undefined, {}, schemaValidation); + // Synthetic member fixtures based on Telerik's SAZ layout; not a real-producer golden. const compressedBody = gzipSync(Buffer.from("stored compressed body")); const members = (): [string, Buffer][] => [ @@ -164,7 +177,7 @@ it("uses HTTP optional whitespace for SAZ identity codings without changing raw ]); else expect(capture.records[index]?.protobuf).toBeUndefined(); } - const { call } = await connectLocalToolsMcp(); + const { call } = await connectSazMcp(); const response = await call("inspect_web_network_capture", { capture_path: path, format: "saz", @@ -241,7 +254,7 @@ it("checks trailer credentials before hidden metadata blocks analysis and withho ]); const path = await fixture(bytes); const sha256 = createHash("sha256").update(bytes).digest("hex"); - const { call } = await connectLocalToolsMcp(); + const { call } = await connectSazMcp(); const base = { capture_path: path, format: "saz", @@ -499,7 +512,9 @@ it("inspects native SAZ through the public CLI and MCP with equal evidence, byte ); expect(Buffer.compare(await readFile(path), bytes)).toBe(0); - const { client, call } = await connectLocalToolsMcp(); + const getValidator = vi.spyOn(schemaValidation, "getValidator"); + onTestFinished(() => getValidator.mockRestore()); + const { client, call } = await connectSazMcp(); const tool = (await client.listTools()).tools.find( (item) => item.name === "inspect_web_network_capture", ); @@ -510,6 +525,13 @@ it("inspects native SAZ through the public CLI and MCP with equal evidence, byte expect(ajv.validate(tool.inputSchema, args)).toBe(true); const response = await call("inspect_web_network_capture", args); expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(getValidator).toHaveBeenCalled(); + const expectedSchema = JSON.stringify(tool.outputSchema); + expect( + getValidator.mock.calls.some( + ([schema]) => JSON.stringify(schema) === expectedSchema, + ), + ).toBe(true); const fromMcp = parseEvidence(response.structuredContent); expect(fromMcp.normalized_result).toEqual(evidence.normalized_result); const bundle = await call("get_evidence_bundle", { detail: "complete" }); @@ -524,7 +546,7 @@ it("inspects native SAZ through the public CLI and MCP with equal evidence, byte it("pages retained SAZ Evidence through public MCP and portable CLI after source deletion", async () => { const path = await fixture(await archive()); - const { call } = await connectLocalToolsMcp(); + const { call } = await connectSazMcp(); const args = { capture_path: path, format: "saz", record_ordinals: [1, 0] }; const full = await call("inspect_web_network_capture", args); const fromMcp = parseEvidence(full.structuredContent); @@ -895,7 +917,7 @@ it.each([ }, }), ]); - const { call } = await connectLocalToolsMcp(); + const { call } = await connectSazMcp(); const response = await call("inspect_web_network_capture", { capture_path: path, format: "saz", diff --git a/tests/boundary/cli/analysisViewCli.test.ts b/tests/boundary/cli/analysisViewCli.test.ts index e5b111c5f4..5b462c3b67 100644 --- a/tests/boundary/cli/analysisViewCli.test.ts +++ b/tests/boundary/cli/analysisViewCli.test.ts @@ -16,7 +16,7 @@ import { expect } from "vitest"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; import { analysisViewLayoutEvidence, @@ -97,7 +97,7 @@ cliTest( "evidence_integrity_mismatch", ], ] as const) { - const rejected = await cli.run({ + const invocation = { arguments: [ "inspect-analysis-view", JSON.stringify({ source, view }), @@ -105,14 +105,18 @@ cliTest( path, "--json", ], - }); + }; + const rejected = + source.kind === "inline" + ? await cli.run(invocation) + : await runFileCli(cli, invocation); expect(rejected.exitCode).toBe(1); expect(rejected.json).toMatchObject({ code: expectedCode }); expect(rejected.json).not.toHaveProperty("evidence_id"); } - for (const [contents, expectedCode] of [ - [JSON.stringify(parent), "invalid_request"], - ["{invalid JSON", "evidence_integrity_mismatch"], + for (const [contents, expectedCode, freshProcess] of [ + [JSON.stringify(parent), "invalid_request", false], + ["{invalid JSON", "evidence_integrity_mismatch", false], [ JSON.stringify({ ...bundle, @@ -122,10 +126,11 @@ cliTest( })), }), "evidence_integrity_mismatch", + true, ], ] as const) { await writeFile(path, contents); - const rejected = await cli.run({ + const invocation = { arguments: [ "inspect-analysis-view", JSON.stringify(request), @@ -133,7 +138,12 @@ cliTest( path, "--json", ], - }); + }; + // Keep a fresh-process integrity check for the altered bundle while the + // worker must reread the same path for the other invalid inputs. + const rejected = freshProcess + ? await cli.run(invocation) + : await runFileCli(cli, invocation); expect(rejected.exitCode).toBe(1); expect(rejected.json).toMatchObject({ code: expectedCode }); expect(rejected.json).not.toHaveProperty("evidence_id"); @@ -416,9 +426,15 @@ cliTest( }; const application = await inspectAnalysisView(request); await writeFile(path, JSON.stringify(request)); - const reply = await cli.run({ + const invocation = { arguments: ["inspect-analysis-view", path, "--json"], - }); + }; + // Exercise a valid and a malformed parent in fresh processes as well + // as the shared worker's repeated reads of the rewritten input path. + const reply = + group === "manifest" && variant !== "missing" + ? await cli.run(invocation) + : await runFileCli(cli, invocation); expect(reply.stderr).toBe(""); if (variant === "valid") { expect(application.ok).toBe(true); diff --git a/tests/boundary/cli/clientProfileLifecycle.test.ts b/tests/boundary/cli/clientProfileLifecycle.test.ts index 71a7e41640..6667e6b186 100644 --- a/tests/boundary/cli/clientProfileLifecycle.test.ts +++ b/tests/boundary/cli/clientProfileLifecycle.test.ts @@ -418,13 +418,12 @@ for (const group of groups) { }); if (client === "grok_build") { const original = await readFile(config, "utf8"); - await writeFile( - config, - original.replace( - "[mcp_servers.rea]", - "[mcp_servers.rea]\ndisabled = true", - ), + const disabled = original.replace( + "[mcp_servers.rea]", + "[mcp_servers.rea]\ndisabled = true", ); + expect(disabled).not.toBe(original); + await writeFile(config, disabled); expect( (await run(["doctor", "--client", "grok_build", "--json"])) .exitCode, diff --git a/tests/boundary/cli/clientProfiles.test.ts b/tests/boundary/cli/clientProfiles.test.ts index 78e8e9c9fa..d8ff64e722 100644 --- a/tests/boundary/cli/clientProfiles.test.ts +++ b/tests/boundary/cli/clientProfiles.test.ts @@ -391,7 +391,6 @@ cliTest( const home = await createTestTempDirectory("rea-gemini-case-"); const system = join(home, "system.json"); const defaults = join(home, "defaults.json"); - await writeFile(system, JSON.stringify({ mcp: { allowed: ["REA"] } })); const environment = { HOME: home, USERPROFILE: home, @@ -404,20 +403,20 @@ cliTest( cwd: home, environment, }); - // The intersection compares names case-insensitively, so the spelling that - // survives must not decide the outcome: the same pair of policies in - // either arrangement has to consolidate to the same decision. + // Keep another server in both policies. Without normalized matching the + // intersection would then exclude REA rather than become an unrestricted + // empty allowlist, so each ordering independently detects that regression. for (const [systemSpelling, defaultSpelling] of [ ["REA", "rea"], ["rea", "REA"], ]) { await writeFile( system, - JSON.stringify({ mcp: { allowed: [systemSpelling] } }), + JSON.stringify({ mcp: { allowed: [systemSpelling, "other"] } }), ); await writeFile( defaults, - JSON.stringify({ mcp: { allowed: [defaultSpelling] } }), + JSON.stringify({ mcp: { allowed: [defaultSpelling, "other"] } }), ); const result = await run(); expect(result.exitCode, `${systemSpelling}+${defaultSpelling}`).toBe(0); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 23ed977b65..1a4e8e171a 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -733,6 +733,65 @@ it("records failed omissions from full reports without treating partial or unhan ).rejects.toMatchObject({ code: "ENOENT" }); }); +it("requires executed Python analysis-efficiency cases while allowing optional skips", async () => { + const root = await createTestTempDirectory("rea-python-analysis-inventory-"); + await mkdir(join(root, "scripts"), { recursive: true }); + await writeFile( + join(root, "scripts", "run-python-tests.py"), + await readFile( + new URL("../../../scripts/run-python-tests.py", import.meta.url), + ), + ); + const passing = + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self): pass\n"; + for (const family of ["mitmproxy", "native", "pwndbg"]) { + await mkdir(join(root, "bridge", family), { recursive: true }); + await writeFile(join(root, "bridge", family, "test_case.py"), passing); + } + const directory = join(root, "tests", "support", "analysis-efficiency"); + await mkdir(directory, { recursive: true }); + const testFile = join(directory, "test_example.py"); + const run = () => + execute("python3", ["scripts/run-python-tests.py"], { cwd: root }); + + await writeFile(join(root, "checkout_support.py"), "VALUE = 42\n"); + await writeFile( + testFile, + "import unittest\nfrom checkout_support import VALUE\nclass Example(unittest.TestCase):\n def test_case(self): self.assertEqual(VALUE, 42)\n", + ); + expect((await run()).stderr).toContain("OK"); + + for (const content of [ + "# A present file can contain no tests.\n", + "import unittest\nclass Example(unittest.TestCase):\n @unittest.skip('unavailable')\n def test_case(self): pass\n", + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n for x in range(2):\n with self.subTest(x=x): self.skipTest('unavailable')\n", + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n with self.subTest(outer=1):\n with self.subTest(inner=2): self.skipTest('unavailable')\n", + ]) { + await writeFile(testFile, content); + await expect(run()).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining( + '"code": "no_executed_analysis_efficiency_tests"', + ), + }); + } + for (const content of [ + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self): pass\n @unittest.skip('optional')\n def test_skipped(self): pass\n", + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n for x in range(2):\n with self.subTest(x=x):\n if x == 0: self.skipTest('optional')\n self.assertEqual(x, 1)\n", + ]) { + await writeFile(testFile, content); + expect((await run()).stderr).toContain("skipped 'optional'"); + } + await writeFile( + testFile, + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n with self.subTest(value=1): self.fail('subtest failed')\n", + ); + await expect(run()).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("subtest failed"), + }); +}); + it("requires executed Python cases in every bridge family", async () => { const root = await createTestTempDirectory("rea-python-inventory-"); await mkdir(join(root, "scripts"), { recursive: true }); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts index 0cb1f7f721..f30e3c724b 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -18,7 +18,10 @@ import { readJavaScriptArtifactFiles, } from "../../../support/javascriptApplicationScope.js"; import { analyzeAndAdoptJavaScriptArtifactFiles } from "../../../../src/application/javascript/JavaScriptArtifactAnalysis.js"; -import { analyzeArtifactFiles } from "../../../fixtures/javascriptArtifactAnalysis.js"; +import { + analyzeArtifactFiles, + withArtifactFileAnalyzer, +} from "../../../fixtures/javascriptArtifactAnalysis.js"; import type { JavaScriptAnalyzedSource } from "../../../../src/domain/javascript/javascriptSourceAnalysis.js"; import { immutableArtifactGraph } from "../../../fixtures/javascriptArtifactGraph.js"; import { createJavaScriptSemanticGraphProjection } from "../../../../src/application/javascript/JavaScriptSemanticGraphBuilder.js"; @@ -404,93 +407,95 @@ it("keeps known IPC pairings when a different Electron value is dynamic", async const snapshot = await scanCanonicalArtifactInventory(root, {}); const reader = createJavaScriptArtifactReader(root, "directory"); try { - const files = await readJavaScriptArtifactFiles(reader, snapshot); - const { analysis } = await analyzeArtifactFiles(files); - expect( - analysis.files.find(({ file }) => file.path === "main.js")?.javascript, - ).toMatchObject({ parse_status: "partial", parse_error_count: 0 }); - const { graph, electronSummary } = await completeJavaScriptAnalysisSteps( - buildImmutableJavaScriptArtifactGraphSteps(snapshot, files, analysis), - ); - expect(graph.coverage.status).toBe("partial"); - expect(electronSummary.ipc).toMatchObject({ - paired_renderer_transmissions: 1, - ambiguous_renderer_transmissions: 0, - }); + await withArtifactFileAnalyzer(async (analyze) => { + const files = await readJavaScriptArtifactFiles(reader, snapshot); + const { analysis } = await analyze(files); + expect( + analysis.files.find(({ file }) => file.path === "main.js")?.javascript, + ).toMatchObject({ parse_status: "partial", parse_error_count: 0 }); + const { graph, electronSummary } = await completeJavaScriptAnalysisSteps( + buildImmutableJavaScriptArtifactGraphSteps(snapshot, files, analysis), + ); + expect(graph.coverage.status).toBe("partial"); + expect(electronSummary.ipc).toMatchObject({ + paired_renderer_transmissions: 1, + ambiguous_renderer_transmissions: 0, + }); - // An unselected source map cannot hide an IPC handler. - const withoutMap = await readJavaScriptArtifactFiles( - reader, - snapshot, - undefined, - ["package.json", "main.js", "preload.js", "hidden.js"], - ); - const withoutMapAnalysis = (await analyzeArtifactFiles(withoutMap)) - .analysis; - const nonJsResult = await completeJavaScriptAnalysisSteps( - buildImmutableJavaScriptArtifactGraphSteps( + // An unselected source map cannot hide an IPC handler. + const withoutMap = await readJavaScriptArtifactFiles( + reader, snapshot, - withoutMap, - withoutMapAnalysis, - ), - ); - expect(nonJsResult.graph.coverage.status).toBe("partial"); - expect(nonJsResult.electronSummary.ipc.paired_renderer_transmissions).toBe( - 1, - ); + undefined, + ["package.json", "main.js", "preload.js", "hidden.js"], + ); + const withoutMapAnalysis = (await analyze(withoutMap)).analysis; + const nonJsResult = await completeJavaScriptAnalysisSteps( + buildImmutableJavaScriptArtifactGraphSteps( + snapshot, + withoutMap, + withoutMapAnalysis, + ), + ); + expect(nonJsResult.graph.coverage.status).toBe("partial"); + expect( + nonJsResult.electronSummary.ipc.paired_renderer_transmissions, + ).toBe(1); - // A source outside the requested scope can hide a competing IPC handler, - // even when the observed sender and handler have a known package owner. - const selected = await readJavaScriptArtifactFiles( - reader, - snapshot, - undefined, - ["package.json", "main.js", "preload.js"], - ); - const selectedAnalysis = (await analyzeArtifactFiles(selected)).analysis; - const selectionResult = await completeJavaScriptAnalysisSteps( - buildImmutableJavaScriptArtifactGraphSteps( + // A source outside the requested scope can hide a competing IPC handler, + // even when the observed sender and handler have a known package owner. + const selected = await readJavaScriptArtifactFiles( + reader, snapshot, - selected, - selectedAnalysis, - ), - ); - expect(selectionResult.electronSummary.ipc).toMatchObject({ - paired_renderer_transmissions: 0, - ambiguous_renderer_transmissions: 1, - }); + undefined, + ["package.json", "main.js", "preload.js"], + ); + const selectedAnalysis = (await analyze(selected)).analysis; + const selectionResult = await completeJavaScriptAnalysisSteps( + buildImmutableJavaScriptArtifactGraphSteps( + snapshot, + selected, + selectedAnalysis, + ), + ); + expect(selectionResult.electronSummary.ipc).toMatchObject({ + paired_renderer_transmissions: 0, + ambiguous_renderer_transmissions: 1, + }); - const hidden = files.files.find(({ path }) => path === "hidden.js"); - if (hidden === undefined) throw new Error("Missing hidden source fixture"); - const unavailable = { - ...files, - files: files.files.filter(({ path }) => path !== hidden.path), - unavailable_relevant_files: [ - ...files.unavailable_relevant_files, - { - path: hidden.path, - container_sha256: hidden.container_sha256, - sha256: hidden.sha256, - bytes: hidden.bytes, - inventory_artifact_id: hidden.inventory_artifact_id, - kind: hidden.kind, - }, - ], - }; - const unavailableAnalysis = { - ...analysis, - files: analysis.files.filter(({ file }) => file.path !== hidden.path), - }; - const unavailableResult = await completeJavaScriptAnalysisSteps( - buildImmutableJavaScriptArtifactGraphSteps( - snapshot, - unavailable, - unavailableAnalysis, - ), - ); - expect(unavailableResult.electronSummary.ipc).toMatchObject({ - paired_renderer_transmissions: 0, - ambiguous_renderer_transmissions: 1, + const hidden = files.files.find(({ path }) => path === "hidden.js"); + if (hidden === undefined) + throw new Error("Missing hidden source fixture"); + const unavailable = { + ...files, + files: files.files.filter(({ path }) => path !== hidden.path), + unavailable_relevant_files: [ + ...files.unavailable_relevant_files, + { + path: hidden.path, + container_sha256: hidden.container_sha256, + sha256: hidden.sha256, + bytes: hidden.bytes, + inventory_artifact_id: hidden.inventory_artifact_id, + kind: hidden.kind, + }, + ], + }; + const unavailableAnalysis = { + ...analysis, + files: analysis.files.filter(({ file }) => file.path !== hidden.path), + }; + const unavailableResult = await completeJavaScriptAnalysisSteps( + buildImmutableJavaScriptArtifactGraphSteps( + snapshot, + unavailable, + unavailableAnalysis, + ), + ); + expect(unavailableResult.electronSummary.ipc).toMatchObject({ + paired_renderer_transmissions: 0, + ambiguous_renderer_transmissions: 1, + }); }); } finally { await reader.close(); @@ -511,34 +516,36 @@ it("keeps IPC pairings uncertain when inline HTML scripts are not examined", asy const snapshot = await scanCanonicalArtifactInventory(root, {}); const reader = createJavaScriptArtifactReader(root, "directory"); try { - const files = await readJavaScriptArtifactFiles(reader, snapshot); - const { analysis } = await analyzeArtifactFiles(files); - expect(analysis.html_inline_scripts.length).toBeGreaterThan(0); - const result = await completeJavaScriptAnalysisSteps( - buildImmutableJavaScriptArtifactGraphSteps(snapshot, files, analysis), - ); - expect(result.electronSummary.ipc).toMatchObject({ - paired_renderer_transmissions: 0, - ambiguous_renderer_transmissions: 1, - }); + await withArtifactFileAnalyzer(async (analyze) => { + const files = await readJavaScriptArtifactFiles(reader, snapshot); + const { analysis } = await analyze(files); + expect(analysis.html_inline_scripts.length).toBeGreaterThan(0); + const result = await completeJavaScriptAnalysisSteps( + buildImmutableJavaScriptArtifactGraphSteps(snapshot, files, analysis), + ); + expect(result.electronSummary.ipc).toMatchObject({ + paired_renderer_transmissions: 0, + ambiguous_renderer_transmissions: 1, + }); - const selected = await readJavaScriptArtifactFiles( - reader, - snapshot, - undefined, - ["package.json", "main.js", "preload.js"], - ); - const selectedAnalysis = (await analyzeArtifactFiles(selected)).analysis; - const selectedResult = await completeJavaScriptAnalysisSteps( - buildImmutableJavaScriptArtifactGraphSteps( + const selected = await readJavaScriptArtifactFiles( + reader, snapshot, - selected, - selectedAnalysis, - ), - ); - expect(selectedResult.electronSummary.ipc).toMatchObject({ - paired_renderer_transmissions: 0, - ambiguous_renderer_transmissions: 1, + undefined, + ["package.json", "main.js", "preload.js"], + ); + const selectedAnalysis = (await analyze(selected)).analysis; + const selectedResult = await completeJavaScriptAnalysisSteps( + buildImmutableJavaScriptArtifactGraphSteps( + snapshot, + selected, + selectedAnalysis, + ), + ); + expect(selectedResult.electronSummary.ipc).toMatchObject({ + paired_renderer_transmissions: 0, + ambiguous_renderer_transmissions: 1, + }); }); } finally { await reader.close(); diff --git a/tests/boundary/filesystem/qoderClientConfiguration.test.ts b/tests/boundary/filesystem/qoderClientConfiguration.test.ts index eec5e594f6..1661c37456 100644 --- a/tests/boundary/filesystem/qoderClientConfiguration.test.ts +++ b/tests/boundary/filesystem/qoderClientConfiguration.test.ts @@ -1,25 +1,12 @@ -import { currentClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; - -import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; import { skillDestinations } from "../../../src/application/SetupSkill.js"; import { supportedClients } from "../../../src/application/SupportedClients.js"; -import { systemUninstallHost } from "../../../src/application/Uninstall.js"; -import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; -/** - * The registration command setup writes on the host running the tests. Windows - * wraps it in `cmd.exe /d /c`, so the expected entry follows the host too. - */ -const command = currentClientRegistrationCommand({ - platform: process.platform, -}); +import { jsonClientRegistrationContract } from "../../support/jsonClientRegistrationContract.js"; beforeEach(clearClientLocationEnvironment); afterEach(() => vi.unstubAllEnvs()); @@ -71,99 +58,11 @@ describe("Qoder configuration paths", () => { }); }); -describe("Qoder JSON registration", () => { - const qoderClient = async () => { - const home = await createTestTempDirectory("rea-qoder-"); - const client = qoderPath(home, process.platform); - if (client?.markerPath === undefined) - throw new Error("missing Qoder client"); - await mkdir(client.markerPath, { recursive: true }); - return { home, client }; - }; - - it("upserts rea into mcpServers while preserving unrelated settings", async () => { - const { home, client } = await qoderClient(); - await writeFile( - client.configPath, - `{ - "mcpServers": { - "other": { "command": "other-server" } - }, - "model": { "name": "qoder-model" } -}`, - ); - - expect( - (await configureClientConfiguration(client, {}, command)).status, - ).toBe("configured"); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.mcpServers.other).toEqual({ command: "other-server" }); - expect(parsed.model).toEqual({ name: "qoder-model" }); - expect(parsed.mcpServers.rea).toEqual({ - command: command[0], - args: command.slice(1), - }); - - expect(await configureClientConfiguration(client, {}, command)).toEqual({ - status: "unchanged", - }); - expect( - ( - await readClientRegistrationStatuses(home, undefined, { - platform: process.platform, - environment: {}, - }) - ).find(({ client }) => client === "qoder"), - ).toMatchObject({ state: "aligned", command }); - }); - - it("creates the mcpServers table in an existing settings file without servers", async () => { - const { client } = await qoderClient(); - await writeFile( - client.configPath, - JSON.stringify({ general: { vimMode: false } }, null, 2), - ); - expect( - (await configureClientConfiguration(client, {}, command)).status, - ).toBe("configured"); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.general).toEqual({ vimMode: false }); - expect(parsed.mcpServers.rea).toEqual({ - command: command[0], - args: command.slice(1), - }); - }); - - it("registers into an empty settings file", async () => { - const { client } = await qoderClient(); - await writeFile(client.configPath, ""); - expect( - (await configureClientConfiguration(client, {}, command)).status, - ).toBe("configured"); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.mcpServers.rea).toEqual({ - command: command[0], - args: command.slice(1), - }); - }); - - it("removes only the rea entry on uninstall", async () => { - const { home, client } = await qoderClient(); - await writeFile( - client.configPath, - JSON.stringify( - { mcpServers: { other: { command: "other-server" } } }, - null, - 2, - ), - ); - await configureClientConfiguration(client, {}, command); - - expect((await systemUninstallHost(home).removeClient(client)).status).toBe( - "removed", - ); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.mcpServers.other).toEqual({ command: "other-server" }); - expect(parsed.mcpServers.rea).toBeUndefined(); - }); +jsonClientRegistrationContract({ + clientName: "qoder", + displayName: "Qoder", + directory: ".qoder", + filename: "settings.json", + initialSettings: { model: { name: "qoder-model" } }, + settingsWithoutServers: { general: { vimMode: false } }, }); diff --git a/tests/boundary/filesystem/workbuddyClientConfiguration.test.ts b/tests/boundary/filesystem/workbuddyClientConfiguration.test.ts index 47736c7d37..1c04dc655f 100644 --- a/tests/boundary/filesystem/workbuddyClientConfiguration.test.ts +++ b/tests/boundary/filesystem/workbuddyClientConfiguration.test.ts @@ -1,25 +1,12 @@ -import { currentClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, win32 } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; - -import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; import { skillDestinations } from "../../../src/application/SetupSkill.js"; import { supportedClients } from "../../../src/application/SupportedClients.js"; -import { systemUninstallHost } from "../../../src/application/Uninstall.js"; -import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; -/** - * The registration command setup writes on the host running the tests. Windows - * wraps it in `cmd.exe /d /c`, so the expected entry follows the host too. - */ -const command = currentClientRegistrationCommand({ - platform: process.platform, -}); +import { jsonClientRegistrationContract } from "../../support/jsonClientRegistrationContract.js"; beforeEach(clearClientLocationEnvironment); afterEach(() => vi.unstubAllEnvs()); @@ -71,97 +58,11 @@ describe("WorkBuddy configuration paths", () => { }); }); -describe("WorkBuddy JSON registration", () => { - const workbuddyClient = async () => { - const home = await createTestTempDirectory("rea-workbuddy-"); - const client = workbuddyPath(home, process.platform); - if (client?.markerPath === undefined) - throw new Error("missing WorkBuddy client"); - await mkdir(client.markerPath, { recursive: true }); - return { home, client }; - }; - - it("upserts rea into mcpServers while preserving unrelated settings", async () => { - const { home, client } = await workbuddyClient(); - await writeFile( - client.configPath, - `{ - "mcpServers": { - "other": { "command": "other-server" } - } -}`, - ); - - expect( - (await configureClientConfiguration(client, {}, command)).status, - ).toBe("configured"); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.mcpServers.other).toEqual({ command: "other-server" }); - expect(parsed.mcpServers.rea).toEqual({ - command: command[0], - args: command.slice(1), - }); - - expect(await configureClientConfiguration(client, {}, command)).toEqual({ - status: "unchanged", - }); - expect( - ( - await readClientRegistrationStatuses(home, undefined, { - platform: process.platform, - environment: {}, - }) - ).find(({ client }) => client === "workbuddy"), - ).toMatchObject({ state: "aligned", command }); - }); - - it("creates the mcpServers table in an existing mcp.json without servers", async () => { - const { client } = await workbuddyClient(); - await writeFile( - client.configPath, - JSON.stringify({ unrelated: { setting: true } }, null, 2), - ); - expect( - (await configureClientConfiguration(client, {}, command)).status, - ).toBe("configured"); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.unrelated).toEqual({ setting: true }); - expect(parsed.mcpServers.rea).toEqual({ - command: command[0], - args: command.slice(1), - }); - }); - - it("registers into an empty mcp.json", async () => { - const { client } = await workbuddyClient(); - await writeFile(client.configPath, ""); - expect( - (await configureClientConfiguration(client, {}, command)).status, - ).toBe("configured"); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.mcpServers.rea).toEqual({ - command: command[0], - args: command.slice(1), - }); - }); - - it("removes only the rea entry on uninstall", async () => { - const { home, client } = await workbuddyClient(); - await writeFile( - client.configPath, - JSON.stringify( - { mcpServers: { other: { command: "other-server" } } }, - null, - 2, - ), - ); - await configureClientConfiguration(client, {}, command); - - expect((await systemUninstallHost(home).removeClient(client)).status).toBe( - "removed", - ); - const parsed = JSON.parse(await readFile(client.configPath, "utf8")); - expect(parsed.mcpServers.other).toEqual({ command: "other-server" }); - expect(parsed.mcpServers.rea).toBeUndefined(); - }); +jsonClientRegistrationContract({ + clientName: "workbuddy", + displayName: "WorkBuddy", + directory: ".workbuddy", + filename: "mcp.json", + initialSettings: {}, + settingsWithoutServers: { unrelated: { setting: true } }, }); diff --git a/tests/boundary/mcp/deferredRuntime.test.ts b/tests/boundary/mcp/deferredRuntime.test.ts index 52463522d9..dbdb207e75 100644 --- a/tests/boundary/mcp/deferredRuntime.test.ts +++ b/tests/boundary/mcp/deferredRuntime.test.ts @@ -9,7 +9,7 @@ import { symlink, writeFile, } from "node:fs/promises"; -import { resolve, join } from "node:path"; +import { resolve, join, dirname } from "node:path"; import { expect, it, onTestFinished } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { GENERATED_MCP_TOOL_CATALOG } from "../../fixtures/mcpToolCatalog.js"; @@ -651,6 +651,33 @@ const copyCompiledPackage = async (prefix: string): Promise => { return directory; }; +// Recorder cleanup tests replace every runtime import through their loader +// hook. Keep only resolvable sentinels so an unmodeled import fails closed. +const createRecorderCleanupFixture = async (): Promise => { + const directory = await createTestTempDirectory("rea-catalog-cleanup-"); + await mkdir(join(directory, "scripts"), { recursive: true }); + await cp( + resolve("scripts/record-mcp-catalog.mjs"), + join(directory, "scripts/record-mcp-catalog.mjs"), + ); + await cp(resolve("package.json"), join(directory, "package.json")); + for (const path of [ + "server/createServer.js", + "composition/binary.js", + "config/parseConfig.js", + "logger.js", + "contracts/promptContracts.js", + ]) { + const file = join(directory, "dist", path); + await mkdir(dirname(file), { recursive: true }); + await writeFile( + file, + `throw new Error(${JSON.stringify(`Fixture import not replaced: ${path}`)});\n`, + ); + } + return directory; +}; + it("records the canonical catalog without a host account or ambient configuration", async () => { const directory = await copyCompiledPackage("rea-catalog-without-account-"); await cp( @@ -705,11 +732,7 @@ it.each([ ])( "preserves producer failure=$producerFails and settles every recorder owner with cleanup failures=$failures", async ({ failures, producerFails }) => { - const directory = await copyCompiledPackage("rea-catalog-cleanup-"); - await cp( - resolve("scripts/record-mcp-catalog.mjs"), - join(directory, "scripts/record-mcp-catalog.mjs"), - ); + const directory = await createRecorderCleanupFixture(); const log = join(directory, "cleanup.jsonl"); const temporary = join(directory, "temporary"); await mkdir(temporary); diff --git a/tests/boundary/mcp/largeToolResult.test.ts b/tests/boundary/mcp/largeToolResult.test.ts index 1c886e5090..023fe7f66b 100644 --- a/tests/boundary/mcp/largeToolResult.test.ts +++ b/tests/boundary/mcp/largeToolResult.test.ts @@ -24,7 +24,11 @@ import { silentLogger } from "../../../src/logger.js"; import { registerOfficialTools } from "../../../src/server/registerOfficialTools.js"; import { createToolResultDelivery } from "../../../src/server/toolResult.js"; import { mcpResponseOverflowProjection } from "../../../src/server/ToolResultDelivery.js"; -import { MINIMUM_RESPONSE_BYTES } from "../../../src/config/mcpResponseBudget.js"; +import { + MINIMUM_RESPONSE_BYTES, + nodeStringCeilingRecovery, + stdioResponseBudgetRecovery, +} from "../../../src/config/mcpResponseBudget.js"; import { MCP_RESULT_STRING_LIMIT, NODE_MAX_STRING_LENGTH, @@ -110,29 +114,16 @@ describe("MCP response budget recovery", () => { effectiveResponseBytes: MINIMUM_RESPONSE_BYTES, nextStep: "Use complete CLI JSON output to consume the analysis.", }); - expect(projected.reason).toContain( - `The effective REA_MCP_MAX_RESPONSE_BYTES value is ${String(MINIMUM_RESPONSE_BYTES)} bytes`, - ); - expect(projected.reason).toContain("floor 10485760 bytes, 10 MiB"); - expect(projected.reason).toContain("restarting the REA process"); - expect(projected.reason).toContain("does not raise"); - expect(projected.reason).toContain("no runtime override"); expect(NODE_MAX_STRING_LENGTH - MCP_RESULT_STRING_LIMIT).toBe(1024); + const guidance = stdioResponseBudgetRecovery( + MINIMUM_RESPONSE_BYTES, + MCP_RESULT_STRING_LIMIT, + NODE_MAX_STRING_LENGTH, + ); + expect(projected.reason).toContain(guidance); + expect(projected.remediationAction).toContain(guidance); expect(projected.reason).toContain( - `REA's reserved single-string limit of ${String(MCP_RESULT_STRING_LIMIT)} UTF-16 code units`, - ); - expect(projected.reason).toContain( - `Node's single-string ceiling of ${String(NODE_MAX_STRING_LENGTH)} UTF-16 code units`, - ); - expect(projected.reason).not.toContain( - `Node's single-string ceiling of ${String(MCP_RESULT_STRING_LIMIT)}`, - ); - expect(projected.reason).not.toContain( - `Node's single-string ceiling is ${String(MCP_RESULT_STRING_LIMIT)}`, - ); - expect(projected.remediationAction).toContain("REA_MCP_MAX_RESPONSE_BYTES"); - expect(projected.remediationAction).not.toContain( - "set REA_MCP_MAX_RESPONSE_BYTES above Node", + "Use complete CLI JSON output to consume the analysis.", ); }); @@ -143,26 +134,14 @@ describe("MCP response budget recovery", () => { effectiveResponseBytes: MINIMUM_RESPONSE_BYTES * 2, nextStep: "Use complete CLI JSON output when the analysis completed.", }); - expect(projected.reason).toContain("REA's reserved single-string limit"); + const guidance = nodeStringCeilingRecovery( + MCP_RESULT_STRING_LIMIT, + NODE_MAX_STRING_LENGTH, + ); + expect(projected.reason).toContain(guidance); + expect(projected.remediationAction).toContain(guidance); expect(projected.reason).toContain( - `REA's reserved single-string limit is ${String(MCP_RESULT_STRING_LIMIT)} UTF-16 code units`, - ); - expect(projected.reason).toContain( - `below Node's single-string ceiling of ${String(NODE_MAX_STRING_LENGTH)} UTF-16 code units`, - ); - expect(projected.reason).not.toContain( - `Node's single-string ceiling is ${String(MCP_RESULT_STRING_LIMIT)}`, - ); - expect(projected.reason).not.toContain( - `Node's single-string ceiling of ${String(MCP_RESULT_STRING_LIMIT)}`, - ); - expect(projected.reason).toContain( - "REA_MCP_MAX_RESPONSE_BYTES does not raise them", - ); - expect(projected.reason).toContain("no runtime override"); - expect(projected.reason).not.toContain("floor 10485760"); - expect(projected.remediationAction).toContain( - "REA_MCP_MAX_RESPONSE_BYTES does not raise them", + "Use complete CLI JSON output when the analysis completed.", ); }); @@ -179,9 +158,12 @@ describe("MCP response budget recovery", () => { }) .parse(delivered.structuredContent).error.remediation.action; expect(action).toContain("get_evidence_bundle and detail: summary"); - expect(action).toContain("REA_MCP_MAX_RESPONSE_BYTES"); expect(action).toContain( - `Node's single-string ceiling of ${String(NODE_MAX_STRING_LENGTH)}`, + stdioResponseBudgetRecovery( + 2048, + MCP_RESULT_STRING_LIMIT, + NODE_MAX_STRING_LENGTH, + ), ); }); @@ -343,12 +325,14 @@ describe("large complete Evidence MCP delivery", () => { }), }) .parse(detached.structuredContent).error; - expect(limits.message).toContain("REA_MCP_MAX_RESPONSE_BYTES"); - expect(limits.message).toContain("10485760"); - expect(limits.message).toContain("restarting the REA process"); - expect(limits.message).toContain("does not raise"); + const guidance = stdioResponseBudgetRecovery( + STDIO_DEFAULT_MAX_BUFFER_SIZE, + MCP_RESULT_STRING_LIMIT, + NODE_MAX_STRING_LENGTH, + ); + expect(limits.message).toContain(guidance); expect(limits.message).not.toContain("Export retained evidence"); - expect(limits.remediation.action).toContain("REA_MCP_MAX_RESPONSE_BYTES"); + expect(limits.remediation.action).toContain(guidance); expect(limits.details.reported_limits).not.toHaveProperty( "evidence_reference", ); diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index f4e9cbfffd..64e55f5972 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -9,6 +9,8 @@ import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { observed } from "../../fixtures/analysisExecution.js"; import { MCP_DOCTOR_SESSION_SCOPE_DETAIL, @@ -22,6 +24,9 @@ import { createCatalogIdentity, } from "../../../src/catalogIdentity.js"; import { compactPresentationForMcpInputSchemaProfile } from "../../../src/config/mcpInputSchemaProfile.js"; +import { parseConfig } from "../../../src/config/parseConfig.js"; +import { runSessionStatus } from "../../../src/application/DirectAnalysisStatus.js"; +import { ok } from "../../../src/domain/result.js"; import { resolveMcpStartupBudgets, MCP_PHASE_BUDGET_FLOOR_MS, @@ -66,9 +71,31 @@ const assertCliCatalogIdentity = async ( } }; +// The CLI commands share this workflow. Exercise each profile here, while +// retaining representative compiled CLI checks for each public command. +const assertStatusCatalogIdentity = async ( + environment: NodeJS.ProcessEnv, + identity: ReturnType, +) => { + const session = createTestBinarySession(() => ({ + close: async () => ok(null), + execute: async () => observed(null), + })); + const status = await runSessionStatus({ + readConfiguration: () => parseConfig(environment), + createBinarySession: () => session, + }); + expect(status).toMatchObject({ + server_identity: { + catalog: { counts: identity.counts, digests: identity.digests }, + }, + }); +}; + const catalogProfiles = [ { name: "default compact input", + cliParity: false, environment: { REA_MCP_INPUT_SCHEMA_PROFILE: undefined, REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", @@ -80,6 +107,7 @@ const catalogProfiles = [ }, { name: "full", + cliParity: true, environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "full", REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", @@ -88,6 +116,7 @@ const catalogProfiles = [ }, { name: "compact", + cliParity: false, environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "compact", REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", @@ -99,6 +128,7 @@ const catalogProfiles = [ }, { name: "output schemas off", + cliParity: false, environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "full", REA_MCP_OUTPUT_SCHEMA_PROFILE: "off", @@ -107,6 +137,7 @@ const catalogProfiles = [ }, { name: "output schemas compact", + cliParity: false, environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "full", REA_MCP_OUTPUT_SCHEMA_PROFILE: "compact", @@ -115,6 +146,7 @@ const catalogProfiles = [ }, { name: "compact input and output schemas off", + cliParity: true, environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "compact", REA_MCP_OUTPUT_SCHEMA_PROFILE: "off", @@ -189,8 +221,8 @@ describe("production MCP doctor", () => { }); it.each(catalogProfiles)( - "verifies the $name catalog through production stdio and both CLI inventories", - async ({ environment, identity }) => { + "verifies the $name catalog through production stdio and status", + async ({ environment, identity, cliParity }) => { const result = await runProductionMcpDoctor({ command: process.execPath, args: [resolve("scripts/rea.mjs"), "mcp"], @@ -223,10 +255,10 @@ describe("production MCP doctor", () => { expect(result.identity?.observed_catalog_digest).toBe( identity.digests.combined_sha256, ); - await assertCliCatalogIdentity( - { ...process.env, ...environment }, - identity, - ); + const selectedEnvironment = { ...process.env, ...environment }; + await assertStatusCatalogIdentity(selectedEnvironment, identity); + if (cliParity) + await assertCliCatalogIdentity(selectedEnvironment, identity); }, 30_000, ); @@ -234,20 +266,22 @@ describe("production MCP doctor", () => { describe("production MCP doctor family scopes", () => { it.each([ - { families: "session", output: "full" }, - { families: "session,application", output: "compact" }, + { families: "session", output: "full", cliParity: false }, + { families: "session,application", output: "compact", cliParity: true }, ] as const)( "reports a selected family scope healthy with its own inventory and digest ($output)", - async ({ families, output }) => { + async ({ families, output, cliParity }) => { + const environment = { + ...process.env, + REA_MCP_TOOL_FAMILIES: families, + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + REA_MCP_OUTPUT_SCHEMA_PROFILE: output, + }; const result = await runProductionMcpDoctor({ command: process.execPath, args: [resolve("scripts/rea.mjs"), "mcp"], cwd: process.cwd(), - environment: { - ...process.env, - REA_MCP_TOOL_FAMILIES: families, - REA_MCP_OUTPUT_SCHEMA_PROFILE: output, - }, + environment, }); const expected = contractsForToolFamilies(new Set(families.split(","))); @@ -264,22 +298,19 @@ describe("production MCP doctor family scopes", () => { }, request_flow: { tool: "binary_session", ok: true }, }); - await assertCliCatalogIdentity( - { - ...process.env, - REA_MCP_TOOL_FAMILIES: families, - REA_MCP_OUTPUT_SCHEMA_PROFILE: output, - }, - catalogIdentityForToolFamilies( - compactPresentationForMcpInputSchemaProfile( - process.env.REA_MCP_INPUT_SCHEMA_PROFILE === "full" - ? "full" - : "compact", - ), - output, - new Set(families.split(",")), - ), + const identity = catalogIdentityForToolFamilies( + compactPresentationForMcpInputSchemaProfile("compact"), + output, + new Set(families.split(",")), ); + expect(result.identity?.expected_catalog_digest).toBe( + identity.digests.combined_sha256, + ); + expect(result.identity?.observed_catalog_digest).toBe( + identity.digests.combined_sha256, + ); + await assertStatusCatalogIdentity(environment, identity); + if (cliParity) await assertCliCatalogIdentity(environment, identity); }, 30_000, ); diff --git a/tests/boundary/mcp/optionalProviderIsolation.test.ts b/tests/boundary/mcp/optionalProviderIsolation.test.ts index 5986c78945..b129c6f9a0 100644 --- a/tests/boundary/mcp/optionalProviderIsolation.test.ts +++ b/tests/boundary/mcp/optionalProviderIsolation.test.ts @@ -2,7 +2,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { McpServer } from "@modelcontextprotocol/server"; -import { afterEach, expect } from "vitest"; +import { afterAll, afterEach, expect } from "vitest"; import { z } from "zod"; import type { BinarySessionPort } from "../../../src/application/binary/BinarySessionPort.js"; @@ -20,6 +20,10 @@ import { createServer } from "../../../src/server/createServer.js"; import { optionalObservationFactories } from "../../fixtures/optionalObservationProviders.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; + +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); const failedAdapters = [ ["browserObservation", "list_browser_targets", "rea-cdp-browser"], @@ -128,7 +132,11 @@ mcpTest.for(failedAdapters)( }); for (const [peer] of failedAdapters) expect(connected.loaded[peer] !== undefined).toBe(peer !== port); - const client = await mcp.connect(connected.server); + const client = await mcp.connect( + connected.server, + undefined, + schemaValidation, + ); const names = (await client.listTools()).tools.map(({ name }) => name); expect(new Set(names)).toEqual( new Set(TOOL_CONTRACTS.map(({ name }) => name)), @@ -252,7 +260,7 @@ mcpTest( electronActiveObservation: failed, javascriptRuntimeObservation: failed, }); - const client = await mcp.connect(first.server); + const client = await mcp.connect(first.server, undefined, schemaValidation); expect((await client.listTools()).tools).toHaveLength( TOOL_CONTRACTS.length, ); diff --git a/tests/boundary/pcap/pcapProvider.test.ts b/tests/boundary/pcap/pcapProvider.test.ts index 184b4ff7be..21f6c83f34 100644 --- a/tests/boundary/pcap/pcapProvider.test.ts +++ b/tests/boundary/pcap/pcapProvider.test.ts @@ -91,37 +91,6 @@ it("inspects both tools without reading a capture", async () => { }); }); -it("rejects a selected TShark whose bytes change during its version probe", async () => { - const root = await createTestTempDirectory("rea-pcap-tool-identity-"); - const { tshark, capinfos } = await writeStubTools(root); - await writeFile( - tshark, - `#!/bin/sh -if [ "$1" = "--version" ]; then - printf 'TShark (Wireshark) 4.6.6-stub\\n' - printf '\\n# changed during version probe\\n' >> "$0" - exit 0 -fi -exit 2 -`, - { mode: 0o755 }, - ); - const provider = new PcapProvider({ - environment: { REA_TSHARK_COMMAND: tshark, REA_CAPINFOS_COMMAND: capinfos }, - }); - onTestFinished(() => provider.close()); - const result = await provider.execute({ - operation: "inspect_pcap_client", - input: {}, - }); - expect(result.ok).toBe(false); - if (!result.ok) - expect(projectAnalysisError(result.error)).toMatchObject({ - code: "artifact_changed", - details: { path: tshark }, - }); -}); - it("inspects a capture with endpoints, DNS, TLS SNI, and HTTP", async () => { const { provider, root } = await providerWith(); const path = await captureFixture(root); @@ -441,16 +410,6 @@ it("reports a non-capture file as unsupported", async () => { }); }); -it("reports a missing tshark binary as a capability gap", async () => { - const provider = new PcapProvider({ - environment: { REA_TSHARK_COMMAND: "/nonexistent/tshark" }, - }); - expect(await provider.inspectAvailability()).toMatchObject({ - status: "unavailable", - code: "executable_missing", - }); -}); - it("rejects a capture that is not a readable file", async () => { const { provider } = await providerWith(); const result = await provider.execute({ @@ -464,25 +423,6 @@ it("rejects a capture that is not a readable file", async () => { }); }); -it("cancels before spawning anything", async () => { - const { provider, root } = await providerWith(); - await writeFile(join(root, "spawned"), ""); - const controller = new AbortController(); - controller.abort(); - const result = await provider.execute( - { - operation: "inspect_pcap_capture", - input: { path: "/some/capture.pcap" }, - }, - { signal: controller.signal }, - ); - expect(result.ok).toBe(false); - if (result.ok) return; - expect(projectAnalysisError(result.error)).toMatchObject({ - code: "cancelled", - }); -}); - it("reports relative configuration and missing capinfos as capability gaps", async () => { const root = await createTestTempDirectory("rea-pcap-config-"); const { tshark } = await writeStubTools(root); diff --git a/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts b/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts index 0c0882c30f..5cc0fc3a4e 100644 --- a/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts +++ b/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts @@ -48,12 +48,13 @@ const capinfosStub = isWindows ? join(stubDir, "capinfos-stub.cmd") : join(stubDir, "capinfos-stub.sh"); -const providerOver = (): PcapProvider => { +const providerOver = (launcher?: PcapCommandLauncher): PcapProvider => { const provider = new PcapProvider({ environment: { REA_TSHARK_COMMAND: tsharkStub, REA_CAPINFOS_COMMAND: capinfosStub, }, + ...(launcher === undefined ? {} : { launcher }), }); onTestFinished(() => provider.close()); return provider; @@ -315,6 +316,7 @@ exit 2 if (!result.ok) expect(projectAnalysisError(result.error)).toMatchObject({ code: "artifact_changed", + details: { path: tshark }, }); }, ); @@ -471,7 +473,10 @@ it("reports a missing binary as a capability gap", async () => { it("cancels before spawning anything", async () => { const controller = new AbortController(); controller.abort(); - const result = await providerOver().execute( + const launcher = vi.fn>(async () => { + throw new Error("Cancelled analysis must not launch a tool"); + }); + const result = await providerOver(launcher).execute( { operation: "inspect_pcap_capture", input: { path: "/any/capture.pcap" } }, { signal: controller.signal }, ); @@ -480,6 +485,7 @@ it("cancels before spawning anything", async () => { expect(projectAnalysisError(result.error)).toMatchObject({ code: "cancelled", }); + expect(launcher).not.toHaveBeenCalled(); }); it.runIf(!hasNativeBound)( diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index c9ca7ca9cf..dfd4edc423 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -1,4 +1,5 @@ import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; import { access, chmod, @@ -17,6 +18,7 @@ import { promisify } from "node:util"; import { _electron as electron } from "playwright-core"; import { expect, it, onTestFinished, vi } from "vitest"; import { z } from "zod"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { @@ -1269,7 +1271,7 @@ it.skipIf(!onDarwin)( timeout: 10_000, }); expect(JSON.parse(stdout)).toEqual({ - argvDecoyIgnored: true, + argvDecoyFailsClosed: true, readableEnvironmentWithoutToken: { runId: null, environmentEntryCount: 1, @@ -1310,16 +1312,35 @@ it.skipIf(!onDarwin)( emptyArgv0Read: true, emptyEnvironmentRecordsBeforeTokenFailClosed: true, duplicateTokenFailsClosed: true, + invalidTokenEncodingFailsClosed: true, finalAssignmentArgumentRead: true, missingAppleBoundaryFailsClosed: true, malformedBuffersRejected: true, + invalidPointerSizeFailsClosed: true, + malformedStages: [ + { state: "unavailable", reason: "malformed_procargs_header" }, + { state: "unavailable", reason: "malformed_procargs_argument_count" }, + { + state: "unavailable", + reason: "malformed_procargs_executable_path", + }, + { state: "unavailable", reason: "malformed_procargs_pointer_size" }, + { state: "unavailable", reason: "malformed_procargs_path_padding" }, + { state: "unavailable", reason: "malformed_procargs_arguments" }, + { state: "unavailable", reason: "malformed_procargs_terminator" }, + { + state: "unavailable", + reason: "malformed_procargs_trailing_entries", + }, + ], malformedObservation: { state: "unavailable", reason: "malformed_procargs", }, nonAsciiPathRead: true, realTokenRead: true, - reservedArgumentIgnored: true, + reservedArgumentWithRealTokenFailsClosed: true, + titleOverwriteFailsClosed: true, truncatedArgvFailsClosed: true, zeroArgumentsRead: true, }); @@ -1439,6 +1460,79 @@ it.skipIf(!onDarwin)( }, ); +it.skipIf(!onDarwin)( + "keeps a long title overwrite unknown when it hides the child's token", + async () => { + const host = createSystemProcessOwnershipHost("darwin"); + const children: ChildProcess[] = []; + try { + for (const length of [256, 384, 512, 1024]) { + const child = spawn( + process.execPath, + [ + "-e", + `if(process.env.PROBE_TITLE!=='-') process.title=process.env.PROBE_TITLE;process.send({ready:true,stillOwned:process.env.REA_PROCESS_RUN_ID==='probe-owned'});setInterval(()=>{},1000)`, + "arg1", + "arg2", + "arg3", + ], + { + env: { + REA_PROCESS_RUN_ID: "probe-owned", + PROBE_TITLE: "n".repeat(length), + PATH: process.env.PATH, + }, + stdio: ["ignore", "ignore", "ignore", "ipc"], + }, + ); + children.push(child); + const message = await waitForChildReadiness(child); + expect(message).toEqual({ ready: true, stillOwned: true }); + if (child.pid === undefined) + throw new Error("test child did not receive a PID"); + const entry = (await host.listProcesses()).find( + ({ pid }) => pid === child.pid, + ); + if ( + entry === undefined || + host.processIdentities === undefined || + host.runTokens === undefined + ) + throw new Error("native child inspection unavailable"); + const before = (await host.processIdentities([entry])).get(child.pid); + const token = (await host.runTokens([entry])).get(child.pid); + const after = (await host.processIdentities([entry])).get(child.pid); + expect(before).toMatchObject({ + state: "readable", + lifecycle: { state: "observed", lifecycle: "non-zombie" }, + }); + if (before?.state !== "readable") + throw new Error("child identity unavailable"); + expect(after).toMatchObject({ + state: "readable", + identity: before.identity, + lifecycle: { state: "observed", lifecycle: "non-zombie" }, + }); + if (token?.state === "readable") { + expect(token.runId).toBe("probe-owned"); + } else { + expect(token).toEqual({ + state: "unavailable", + reason: "ambiguous_environment_boundary", + }); + } + } + } finally { + try { + await Promise.all(children.map(stopNodeChild)); + } finally { + await host.close?.(); + } + } + }, + 150_000, +); + it.skipIf(!onDarwin)( "reads owned tokens across executable padding and distinguishes a real empty environment", async () => { @@ -1517,6 +1611,36 @@ const stopNodeChild = async (child: ChildProcess) => { await exited; }; +const waitForChildReadiness = async (child: ChildProcess): Promise => { + const listeners = new AbortController(); + const message = once(child, "message", { signal: listeners.signal }); + const exit = once(child, "exit", { signal: listeners.signal }); + try { + await awaitAcknowledgement( + message.then(() => undefined), + exit, + "title-mutating child readiness", + ); + return (await message)[0]; + } finally { + listeners.abort(); + } +}; + +it("stops a child that never acknowledges readiness", async () => { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + stdio: ["ignore", "ignore", "ignore", "ipc"], + }); + try { + await expect(waitForChildReadiness(child)).rejects.toThrow( + "title-mutating child readiness was not acknowledged before the gate deadline", + ); + } finally { + await stopNodeChild(child); + } + expect(child.exitCode !== null || child.signalCode !== null).toBe(true); +}); + it.skipIf(process.platform === "win32").each([ { name: "invalid UTF-8", diff --git a/tests/conformance/contracts/mcpResponseBudget.test.ts b/tests/conformance/contracts/mcpResponseBudget.test.ts index f19585eff3..75c144f02f 100644 --- a/tests/conformance/contracts/mcpResponseBudget.test.ts +++ b/tests/conformance/contracts/mcpResponseBudget.test.ts @@ -27,26 +27,28 @@ describe("MCP response budget configuration", () => { reserved, nodeCeiling, ); - expect(recovery).toContain( - "The effective REA_MCP_MAX_RESPONSE_BYTES value is 20971520 bytes", - ); - expect(recovery).toContain("floor 10485760 bytes, 10 MiB"); - expect(recovery).toContain("restarting the REA process"); - expect(recovery).toContain( - `REA's reserved single-string limit of ${String(reserved)} UTF-16 code units`, - ); - expect(recovery).toContain( - `Node's single-string ceiling of ${String(nodeCeiling)} UTF-16 code units`, - ); - expect(recovery).not.toContain( - `Node's single-string ceiling of ${String(reserved)}`, - ); - expect(recovery).not.toContain( - `Node's single-string ceiling is ${String(reserved)}`, - ); - expect(nodeStringCeilingRecovery(reserved, nodeCeiling)).toBe( - `REA's reserved single-string limit is ${String(reserved)} UTF-16 code units, below Node's single-string ceiling of ${String(nodeCeiling)} UTF-16 code units. This release has no runtime override for either limit, and REA_MCP_MAX_RESPONSE_BYTES does not raise them.`, - ); + expect(recovery).toMatch(/REA_MCP_MAX_RESPONSE_BYTES[^.]*20971520 bytes/u); + expect(recovery).toMatch(/floor 10485760 bytes, 10 MiB/u); + expect(recovery).toMatch(/restart\w* the REA process/u); + const nodeRecovery = nodeStringCeilingRecovery(reserved, nodeCeiling); + expect(nodeRecovery).toMatch(/below Node's single-string ceiling/u); + for (const guidance of [recovery, nodeRecovery]) { + expect(guidance).toMatch( + new RegExp( + `REA's reserved single-string limit [^.]*${String(reserved)} UTF-16 code units`, + "u", + ), + ); + expect(guidance).toMatch( + new RegExp( + `Node's single-string ceiling [^.]*${String(nodeCeiling)} UTF-16 code units`, + "u", + ), + ); + expect(guidance).toContain("REA_MCP_MAX_RESPONSE_BYTES"); + expect(guidance).toMatch(/does not raise/u); + expect(guidance).toMatch(/no runtime override/u); + } }); it.each(["", "1.5", "1e8", " 10485760", "9007199254740992"])( diff --git a/tests/fixtures/javascriptArtifactAnalysis.ts b/tests/fixtures/javascriptArtifactAnalysis.ts index 9abfcfad0a..327554f539 100644 --- a/tests/fixtures/javascriptArtifactAnalysis.ts +++ b/tests/fixtures/javascriptArtifactAnalysis.ts @@ -5,36 +5,56 @@ import { type JavaScriptSemanticGraphProjection, } from "../../src/application/javascript/JavaScriptSemanticGraphBuilder.js"; import { createJavaScriptSourceAnalysis } from "../../src/artifacts/javascript/JavaScriptSourceAnalysis.js"; +import type { ArtifactResourceScope } from "../../src/artifacts/ArtifactResourceScope.js"; import type { JavaScriptArtifactFileSet } from "../../src/domain/javascript/javascriptArtifactFiles.js"; import { withArtifactResourceScope } from "./artifactInventory.js"; /** Analyze inert fixture sources through the real worker and join its owning scope. */ -export const analyzeArtifactFiles = ( +const analyzeArtifactFilesInScope = async ( fileSet: JavaScriptArtifactFileSet, + scope: ArtifactResourceScope, ): Promise<{ readonly analysis: JavaScriptArtifactAnalysis; readonly projection: JavaScriptSemanticGraphProjection; -}> => - withArtifactResourceScope(async (scope) => { - const worker = await createJavaScriptSourceAnalysis( - { heapMb: 512, timeoutMs: 30_000 }, - fileSet.files.filter( - (file) => file.kind === "javascript" && file.text.included, - ).length, - { scope }, - ); - scope.retain({ - kind: "javascript-source-analysis", - resource: "fixture JavaScript source analyzer", - analysis: worker, - }); - const projection = createJavaScriptSemanticGraphProjection(); - const outcome = await analyzeAndAdoptJavaScriptArtifactFiles( - fileSet, - worker, - projection, - async () => {}, - ); - if (outcome.interruption !== null) throw outcome.interruption; - return { analysis: outcome.analysis, projection }; - }); +}> => { + const worker = await createJavaScriptSourceAnalysis( + { heapMb: 512, timeoutMs: 30_000 }, + fileSet.files.filter( + (file) => file.kind === "javascript" && file.text.included, + ).length, + { scope }, + ); + const owner = { + kind: "javascript-source-analysis", + resource: "fixture JavaScript source analyzer", + analysis: worker, + } as const; + scope.retain(owner); + const projection = createJavaScriptSemanticGraphProjection(); + const outcome = await analyzeAndAdoptJavaScriptArtifactFiles( + fileSet, + worker, + projection, + async () => {}, + ); + if (outcome.interruption !== null) throw outcome.interruption; + const released = await scope.release(owner); + if (released.kind === "failed") + throw scope.cleanupFailure({ kind: "completed" }, [released]); + return { analysis: outcome.analysis, projection }; +}; + +/** Reuse a healthy worker across analyses owned by the same test. */ +export const withArtifactFileAnalyzer = ( + operation: ( + analyze: ( + fileSet: JavaScriptArtifactFileSet, + ) => ReturnType, + ) => Promise, +): Promise => + withArtifactResourceScope((scope) => + operation((fileSet) => analyzeArtifactFilesInScope(fileSet, scope)), + ); + +export const analyzeArtifactFiles = (fileSet: JavaScriptArtifactFileSet) => + withArtifactFileAnalyzer((analyze) => analyze(fileSet)); diff --git a/tests/fixtures/javascriptReturnShapes.ts b/tests/fixtures/javascriptReturnShapes.ts index e26f0d35d5..d796cd4a2c 100644 --- a/tests/fixtures/javascriptReturnShapes.ts +++ b/tests/fixtures/javascriptReturnShapes.ts @@ -36,13 +36,12 @@ const exportedReturnFields = (value: unknown) => { /** Select complete return fields from a real JavaScript application result. */ export type JavaScriptReturnFields = ReturnType; -/** Exercise identical return assertions through the compiled CLI and stdio MCP. */ -export const verifyJavaScriptReturnShapes = async ( +const analyzeReturnShapesThroughCli = async ( cli: TestCli, source: string, check: (fields: JavaScriptReturnFields) => void, - fileName = "app.js", -): Promise => { + fileName: string, +) => { const root = await createTestTempDirectory("rea-return-shapes-"); await writeFile(join(root, fileName), source); const cliResponse = await cli.run({ @@ -58,6 +57,31 @@ export const verifyJavaScriptReturnShapes = async ( const contract = toolContract("analyze_javascript_application"); const cliEvidence = contract.outputSchema.parse(cliResponse.json); check(exportedReturnFields(cliEvidence.normalized_result)); + return { root, contract, cliEvidence }; +}; + +/** Exercise return assertions through the compiled CLI and its real worker. */ +export const verifyJavaScriptReturnShapesThroughCli = async ( + cli: TestCli, + source: string, + check: (fields: JavaScriptReturnFields) => void, +): Promise => { + await analyzeReturnShapesThroughCli(cli, source, check, "app.js"); +}; + +/** Exercise identical return assertions through the compiled CLI and stdio MCP. */ +export const verifyJavaScriptReturnShapes = async ( + cli: TestCli, + source: string, + check: (fields: JavaScriptReturnFields) => void, + fileName = "app.js", +): Promise => { + const { root, contract, cliEvidence } = await analyzeReturnShapesThroughCli( + cli, + source, + check, + fileName, + ); const client = new Client({ name: "return-shapes-e2e", version: "1" }); const transport = new StdioClientTransport({ command: process.execPath, diff --git a/tests/fixtures/localToolsMcp.ts b/tests/fixtures/localToolsMcp.ts index 9979939c9a..e2de6357d2 100644 --- a/tests/fixtures/localToolsMcp.ts +++ b/tests/fixtures/localToolsMcp.ts @@ -1,6 +1,7 @@ import { Client, type CallToolRequestOptions, + type ClientOptions, } from "@modelcontextprotocol/client"; import { StdioClientTransport, @@ -41,6 +42,7 @@ export async function connectLocalToolsMcp( environment: Readonly> = {}, registerCleanup: (cleanup: () => Promise) => void = onTestFinished, transportOptions: Readonly> = {}, + jsonSchemaValidator?: NonNullable, ) { const transport = new StdioClientTransport({ ...transportOptions, @@ -57,7 +59,10 @@ export async function connectLocalToolsMcp( }, stderr: "pipe", }); - const client = new Client({ name: "local-tools-e2e", version: "1" }); + const client = new Client( + { name: "local-tools-e2e", version: "1" }, + jsonSchemaValidator === undefined ? undefined : { jsonSchemaValidator }, + ); registerCleanup(async () => { await client.close(); await transport.close(); diff --git a/tests/fixtures/processRunTokenParserProbe.swift b/tests/fixtures/processRunTokenParserProbe.swift index 010f8a62d3..ed359869d8 100644 --- a/tests/fixtures/processRunTokenParserProbe.swift +++ b/tests/fixtures/processRunTokenParserProbe.swift @@ -26,10 +26,29 @@ func fixture( @main struct ProcessRunTokenParserProbe { static func main() throws { - let argvDecoy = try parseProcessEnvironment(in: fixture( - arguments: ["/fixture/executable", "REA_PROCESS_RUN_ID=argv-decoy", "trailing"], - environment: ["DECOY=words REA_PROCESS_RUN_ID=synthetic-decoy"] - )) + let argvDecoyFailsClosed: Bool + do { + _ = try parseProcessEnvironment(in: fixture( + arguments: ["/fixture/executable", "REA_PROCESS_RUN_ID=argv-decoy", "trailing"], + environment: ["DECOY=words REA_PROCESS_RUN_ID=synthetic-decoy"] + )) + argvDecoyFailsClosed = false + } catch RunTokenReadError.ambiguousEnvironmentBoundary { + argvDecoyFailsClosed = true + } + // This layout matches the observed long-title overwrite: a former + // environment entry is consumed as an argv slot before the cleared pfz. + let titleOverwriteFailsClosed: Bool + do { + _ = try parseProcessEnvironment(in: fixture( + arguments: [String(repeating: "n", count: 248), "REA_PROCESS_RUN_ID=owned-token", "a", "b", "c", "d"], + environment: ["PATH=/usr/bin"], + zeroedPfz: true + )) + titleOverwriteFailsClosed = false + } catch RunTokenReadError.ambiguousEnvironmentBoundary { + titleOverwriteFailsClosed = true + } let realToken = try parseProcessEnvironment(in: fixture( arguments: ["/fixture/executable"], environment: ["DECOY=words REA_PROCESS_RUN_ID=synthetic-decoy", "REA_PROCESS_RUN_ID=owned-token"] @@ -57,10 +76,16 @@ struct ProcessRunTokenParserProbe { arguments: ["/fixture/executable", "FOO=bar"], environment: ["REA_PROCESS_RUN_ID=final-assignment-token"] )) - let reservedArgumentIgnored = try parseProcessEnvironment(in: fixture( - arguments: ["/fixture/executable", "REA_PROCESS_RUN_ID=argv-decoy"], - environment: ["DECOY=words REA_PROCESS_RUN_ID=synthetic-decoy"] - )) + let reservedArgumentWithRealTokenFailsClosed: Bool + do { + _ = try parseProcessEnvironment(in: fixture( + arguments: ["/fixture/executable", "REA_PROCESS_RUN_ID=argv-decoy"], + environment: ["REA_PROCESS_RUN_ID=actual-token"] + )) + reservedArgumentWithRealTokenFailsClosed = false + } catch RunTokenReadError.ambiguousEnvironmentBoundary { + reservedArgumentWithRealTokenFailsClosed = true + } let nonAsciiPathRead = try parseProcessEnvironment(in: fixture( arguments: ["/fixture/é", "FOO=bar"], environment: ["REA_PROCESS_RUN_ID=utf8-path-token"], @@ -123,30 +148,46 @@ struct ProcessRunTokenParserProbe { do { _ = try parseProcessEnvironment(in: truncatedArgv) truncatedArgvFailsClosed = false - } catch RunTokenReadError.invalidBuffer { + } catch RunTokenReadError.invalidBufferAt(.arguments) { truncatedArgvFailsClosed = true } var invalidPadding = fixture(arguments: ["x"], environment: [], executablePath: "x") invalidPadding[6] = 1 // argc (4 bytes), path and its NUL precede the padding. - var unterminated = fixture(arguments: ["x"], environment: [], executablePath: "x") + var unterminated = fixture(arguments: ["REA_PROCESS_RUN_ID=argv-decoy"], environment: [], executablePath: "x") unterminated.removeLast() - let malformedBuffers: [[UInt8]] = [ - [0, 0, 0], // Short argc. - [255, 255, 255, 255], // Negative argc. - [0, 0, 0, 0, 120], // Unterminated executable path. - invalidPadding, - unterminated, + let malformedBuffers: [([UInt8], ProcargsMalformedStage)] = [ + ([0, 0, 0], .header), + ([255, 255, 255, 255], .argumentCount), + ([0, 0, 0, 0, 120], .executablePath), + (invalidPadding, .pathPadding), + (unterminated, .terminator), + (truncatedArgv, .arguments), ] - let malformedBuffersRejected = malformedBuffers.allSatisfy { bytes in + let malformedBuffersRejected = malformedBuffers.allSatisfy { bytes, expectedStage in do { _ = try parseProcessEnvironment(in: bytes) return false - } catch RunTokenReadError.invalidBuffer { - return true + } catch RunTokenReadError.invalidBufferAt(let stage) { + return stage == expectedStage } catch { return false } } + let invalidPointerSizeFailsClosed: Bool + do { + _ = try parseProcessEnvironment(in: fixture(arguments: [], environment: []), pointerSize: 3) + invalidPointerSizeFailsClosed = false + } catch RunTokenReadError.invalidBufferAt(.pointerSize) { + invalidPointerSizeFailsClosed = true + } + let malformedStages = ProcargsMalformedStage.allCases.map { stage in + let observation = processRunTokenObservation( + pid: 1, + parse: .failure(.invalidBufferAt(stage)), + codeSigningFlags: { 0 } + ) + return ["state": observation.state, "reason": observation.reason ?? ""] + } let malformedObservation = processRunTokenObservation( pid: 1, parse: .failure(.invalidBuffer), @@ -155,13 +196,25 @@ struct ProcessRunTokenParserProbe { let duplicateTokenFailsClosed: Bool do { _ = try parseProcessEnvironment(in: fixture( - arguments: ["/fixture/executable"], + arguments: ["/fixture/executable", "REA_PROCESS_RUN_ID=argv-decoy"], environment: ["REA_PROCESS_RUN_ID=first", "REA_PROCESS_RUN_ID=second"] )) duplicateTokenFailsClosed = false } catch RunTokenReadError.duplicateToken { duplicateTokenFailsClosed = true } + var invalidTokenEncoding = fixture( + arguments: ["/fixture/executable", "REA_PROCESS_RUN_ID=argv-decoy"], + environment: ["REA_PROCESS_RUN_ID=é"] + ) + invalidTokenEncoding[invalidTokenEncoding.firstIndex(of: 0xc3)!] = 0xff + let invalidTokenEncodingFailsClosed: Bool + do { + _ = try parseProcessEnvironment(in: invalidTokenEncoding) + invalidTokenEncodingFailsClosed = false + } catch RunTokenReadError.invalidTokenEncoding { + invalidTokenEncodingFailsClosed = true + } let missingAppleBoundaryFailsClosed: Bool do { _ = try parseProcessEnvironment(in: fixture( @@ -217,7 +270,8 @@ struct ProcessRunTokenParserProbe { ) let result: [String: Any] = [ - "argvDecoyIgnored": argvDecoy.runId == nil && argvDecoy.environmentEntryCount == 1, + "argvDecoyFailsClosed": argvDecoyFailsClosed, + "titleOverwriteFailsClosed": titleOverwriteFailsClosed, "realTokenRead": realToken.runId == "owned-token", "readableEnvironmentWithoutToken": [ "runId": readableNoToken.runId as Any? ?? NSNull(), @@ -255,15 +309,18 @@ struct ProcessRunTokenParserProbe { "alternatePaddingRead": alternatePadding.runId == "alternate-padding-token", "truncatedArgvFailsClosed": truncatedArgvFailsClosed, "malformedBuffersRejected": malformedBuffersRejected, + "invalidPointerSizeFailsClosed": invalidPointerSizeFailsClosed, + "malformedStages": malformedStages, "malformedObservation": [ "state": malformedObservation.state, "reason": malformedObservation.reason as Any? ?? NSNull(), ], "duplicateTokenFailsClosed": duplicateTokenFailsClosed, + "invalidTokenEncodingFailsClosed": invalidTokenEncodingFailsClosed, "missingAppleBoundaryFailsClosed": missingAppleBoundaryFailsClosed, "emptyArgv0Read": emptyArgv0Read.runId == "empty-argv0-token", "finalAssignmentArgumentRead": finalAssignmentArgumentRead.runId == "final-assignment-token", - "reservedArgumentIgnored": reservedArgumentIgnored.runId == nil && reservedArgumentIgnored.environmentEntryCount == 1, + "reservedArgumentWithRealTokenFailsClosed": reservedArgumentWithRealTokenFailsClosed, "nonAsciiPathRead": nonAsciiPathRead.runId == "utf8-path-token", "appleVectorTokenFailsClosed": appleVectorTokenFailsClosed, "clearedAppleVectorTokenFailsClosed": clearedAppleVectorTokenFailsClosed, diff --git a/tests/support/jsonClientRegistrationContract.ts b/tests/support/jsonClientRegistrationContract.ts new file mode 100644 index 0000000000..e0c09bd2b8 --- /dev/null +++ b/tests/support/jsonClientRegistrationContract.ts @@ -0,0 +1,126 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { currentClientRegistrationCommand } from "../../src/application/ClientRegistrationIdentity.js"; +import { readClientRegistrationStatuses } from "../../src/application/ClientRegistrationStatus.js"; +import { configureClientConfiguration } from "../../src/application/SetupClientConfiguration.js"; +import { supportedClients } from "../../src/application/SupportedClients.js"; +import { systemUninstallHost } from "../../src/application/Uninstall.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +/** Run the JSON registration contract against each client's independently supplied location. */ +export const jsonClientRegistrationContract = ({ + clientName, + displayName, + directory, + filename, + initialSettings, + settingsWithoutServers, +}: { + clientName: "qoder" | "workbuddy"; + displayName: string; + directory: string; + filename: string; + initialSettings: Record; + settingsWithoutServers: Record; +}): void => { + const command = currentClientRegistrationCommand({ + platform: process.platform, + }); + + describe(`${displayName} JSON registration`, () => { + const createClient = async () => { + const home = await createTestTempDirectory(`rea-${clientName}-`); + const client = supportedClients(home, process.platform, {}).find( + ({ name }) => name === clientName, + ); + if (client?.markerPath === undefined) + throw new Error(`missing ${displayName} client`); + expect(client.markerPath).toBe(join(home, directory)); + expect(client.configPath).toBe(join(home, directory, filename)); + await mkdir(client.markerPath, { recursive: true }); + return { home, client }; + }; + + it("upserts rea into mcpServers while preserving unrelated settings", async () => { + const { home, client } = await createClient(); + await writeFile( + client.configPath, + JSON.stringify({ + ...initialSettings, + mcpServers: { other: { command: "other-server" } }, + }), + ); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + for (const [key, value] of Object.entries(initialSettings)) + expect(parsed[key]).toEqual(value); + expect(parsed.mcpServers.other).toEqual({ command: "other-server" }); + expect(parsed.mcpServers.rea).toEqual({ + command: command[0], + args: command.slice(1), + }); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "unchanged", + }); + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client }) => client === clientName), + ).toMatchObject({ state: "aligned", command }); + }); + + it("creates the mcpServers table in existing settings without servers", async () => { + const { client } = await createClient(); + await writeFile( + client.configPath, + JSON.stringify(settingsWithoutServers), + ); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + for (const [key, value] of Object.entries(settingsWithoutServers)) + expect(parsed[key]).toEqual(value); + expect(parsed.mcpServers.rea).toEqual({ + command: command[0], + args: command.slice(1), + }); + }); + + it("registers into an empty settings file", async () => { + const { client } = await createClient(); + await writeFile(client.configPath, ""); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + expect(parsed.mcpServers.rea).toEqual({ + command: command[0], + args: command.slice(1), + }); + }); + + it("removes only the rea entry on uninstall", async () => { + const { home, client } = await createClient(); + await writeFile( + client.configPath, + JSON.stringify({ mcpServers: { other: { command: "other-server" } } }), + ); + await configureClientConfiguration(client, {}, command); + expect( + (await systemUninstallHost(home).removeClient(client)).status, + ).toBe("removed"); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + expect(parsed.mcpServers.other).toEqual({ command: "other-server" }); + expect(parsed.mcpServers.rea).toBeUndefined(); + }); + }); +};