From a5bfd0440173cc4e3cb6c726999b5f74fc87be9e Mon Sep 17 00:00:00 2001 From: Adolanium <94890352+Adolanium@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:37:20 +0300 Subject: [PATCH 01/37] fix(hopper): keep installed apt dependencies at their versions during Linux setup (#6974) * fix(hopper): keep installed apt dependencies at their versions during Linux setup apt-get install upgrades every listed package that is already installed, so Linux setup could upgrade the system python3 and X11 libraries while installing Hopper. Runtime dependencies are now installed with --no-upgrade in a separate transaction before the Hopper archive, which may still upgrade. The setup approval plan lists both commands. * fix(hopper): narrow the apt upgrade claim and cover split transaction failures * fix(evidence): preserve JSON ownership through metadata updates --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/installation.md | 6 ++ src/application/LinuxHopper.test.ts | 26 +++++ src/application/LinuxHopper.ts | 7 +- .../LinuxHopperPackageRuntime.test.ts | 97 +++++++++++++++---- 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 13ce21d257..8535e2cec1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -681,6 +681,12 @@ search permissions (`0700`) on a directory you own before retrying. On supported Linux distributions, approved setup verifies Hopper's official `.deb`, `.rpm`, or Arch package before invoking the native package manager. +On apt systems, Hopper's runtime dependencies are installed with +`--no-upgrade` in their own transaction, so setup no longer names an +already-installed dependency as an upgrade target. apt can still upgrade a +package that the Hopper package or the Python library repair requires at a +newer version. Arch's `pacman -S --needed` still updates an outdated +dependency. The official release feed selects the current public demo without a version pin. REA verifies the package's detached PGP signature against Hopper's stable primary fingerprint `7C2D73BAFAF5B9A710A69DA1A26DF815099B1BCD`, then checks diff --git a/src/application/LinuxHopper.test.ts b/src/application/LinuxHopper.test.ts index b43d048c9c..b9a1f80624 100644 --- a/src/application/LinuxHopper.test.ts +++ b/src/application/LinuxHopper.test.ts @@ -277,6 +277,32 @@ describe("Linux Hopper installation", () => { ); }, ); + + it.each([true, false])( + "keeps installed apt dependencies at their versions while Hopper may upgrade (root: %s)", + (isRoot) => { + const prefix = isRoot ? [] : ["pkexec"]; + expect( + linuxPackageManagerCommands("deb", "/tmp/Hopper package", isRoot).map( + ({ executable, args }) => [executable, ...args], + ), + ).toEqual([ + [ + ...prefix, + "apt-get", + "install", + "-y", + "--no-upgrade", + "xvfb", + "xauth", + "python3", + "libx11-6", + "libxtst6", + ], + [...prefix, "apt-get", "install", "-y", "/tmp/Hopper package"], + ]); + }, + ); }); describe("Linux Hopper package installation", () => { diff --git a/src/application/LinuxHopper.ts b/src/application/LinuxHopper.ts index a401a4c13a..871b5be48f 100644 --- a/src/application/LinuxHopper.ts +++ b/src/application/LinuxHopper.ts @@ -818,7 +818,12 @@ export const linuxPackageManagerCommands = ( : ["xorg-server-xvfb", "xorg-xauth", "python", "libx11", "libxtst"]; const commands = family === "deb" - ? [["apt-get", "install", "-y", archive, ...runtimeDependencies]] + ? [ + // apt-get install upgrades listed packages that are already + // installed, so dependencies are not named as upgrade targets. + ["apt-get", "install", "-y", "--no-upgrade", ...runtimeDependencies], + ["apt-get", "install", "-y", archive], + ] : family === "rpm" ? [["dnf", "install", "-y", archive, ...runtimeDependencies]] : [ diff --git a/src/application/LinuxHopperPackageRuntime.test.ts b/src/application/LinuxHopperPackageRuntime.test.ts index 359c51f81a..b3cda7929f 100644 --- a/src/application/LinuxHopperPackageRuntime.test.ts +++ b/src/application/LinuxHopperPackageRuntime.test.ts @@ -6,9 +6,17 @@ import { err, ok } from "../domain/result.js"; import * as pythonRuntime from "./LinuxHopperPython.js"; import { linuxHopperInstallDisclosure, + linuxPackageManagerCommands, systemLinuxHopperInstallHost, } from "./LinuxHopper.js"; +/** Package transactions that install Hopper before any runtime library repair. */ +const debCommands = linuxPackageManagerCommands( + "deb", + "/tmp/verified-hopper.deb", + false, +).length; + const missingPython = { interpreter: "/usr/bin/python3", version: [3, 14], @@ -37,13 +45,13 @@ describe("Linux Hopper package runtime", () => { { name: "installs and verifies the matching future Python library", inspections: [ok(missingPython), ok(readyPython)], - packageCalls: 2, + packageCalls: debCommands + 1, status: "installed", }, { name: "skips the extra APT command for an already loadable library", inspections: [ok(readyPython)], - packageCalls: 1, + packageCalls: debCommands, status: "installed", }, { @@ -54,13 +62,13 @@ describe("Linux Hopper package runtime", () => { library: { status: "unknown", reason: "No INSTSONAME" }, } satisfies pythonRuntime.LinuxHopperPythonRuntime), ], - packageCalls: 1, + packageCalls: debCommands, status: "runtime_unavailable", }, { name: "rejects APT success when the library still cannot load", inspections: [ok(missingPython), ok(missingPython)], - packageCalls: 2, + packageCalls: debCommands + 1, status: "runtime_unavailable", }, { @@ -73,7 +81,7 @@ describe("Linux Hopper package runtime", () => { ), ), ], - packageCalls: 1, + packageCalls: debCommands, status: "runtime_unavailable", }, ])("$name", async ({ inspections, packageCalls, status }) => { @@ -97,9 +105,9 @@ describe("Linux Hopper package runtime", () => { ).toMatchObject({ status }); expect(command).toHaveBeenCalledTimes(packageCalls); expect(inspect).toHaveBeenCalledTimes(inspections.length); - if (packageCalls === 2) { + if (packageCalls > debCommands) { const isRoot = process.getuid?.() === 0; - expect(command.mock.calls[1]?.slice(0, 2)).toEqual([ + expect(command.mock.calls[debCommands]?.slice(0, 2)).toEqual([ isRoot ? "apt-get" : "pkexec", [...(isRoot ? [] : ["apt-get"]), "install", "-y", "libpython3.14"], ]); @@ -128,10 +136,10 @@ describe("Linux Hopper package runtime", () => { stderr: "original package error", outputTruncated: true, }; - const command = vi - .spyOn(boundedCommands, "execFileBoundedOutput") - .mockResolvedValueOnce(commandSuccess) - .mockResolvedValueOnce(failed); + const command = vi.spyOn(boundedCommands, "execFileBoundedOutput"); + for (let index = 0; index < debCommands; index++) + command.mockResolvedValueOnce(commandSuccess); + command.mockResolvedValueOnce(failed); const inspect = vi .spyOn(pythonRuntime, "inspectLinuxHopperPythonRuntime") .mockResolvedValue(ok(missingPython)); @@ -139,7 +147,7 @@ describe("Linux Hopper package runtime", () => { expect( await host.installPackage("deb", "/tmp/verified-hopper.deb"), ).toMatchObject({ status: "failed", failure: failed }); - expect(command).toHaveBeenCalledTimes(2); + expect(command).toHaveBeenCalledTimes(debCommands + 1); expect(inspect).toHaveBeenCalledTimes(1); } finally { try { @@ -184,7 +192,7 @@ describe("Linux Hopper package mapping", () => { message: expect.stringContaining(path), }, }); - expect(command).toHaveBeenCalledTimes(1); + expect(command).toHaveBeenCalledTimes(debCommands); expect(inspect).toHaveBeenCalledTimes(1); } finally { try { @@ -197,13 +205,62 @@ describe("Linux Hopper package mapping", () => { }); }); +describe("Linux Hopper package transaction failures", () => { + it.each(Array.from({ length: debCommands }, (_, index) => index + 1))( + "stops after failed install transaction %i without inspecting Python", + async (failing) => { + const directory = await createTestTempDirectory( + "rea-linux-install-failure-", + ); + const host = systemLinuxHopperInstallHost( + { XDG_CACHE_HOME: directory }, + async () => undefined, + ); + const failed = { + exitCode: 100, + stdout: "", + stderr: "transaction " + String(failing) + " failed", + outputTruncated: false, + }; + let calls = 0; + const command = vi + .spyOn(boundedCommands, "execFileBoundedOutput") + .mockImplementation(async () => + ++calls === failing ? failed : commandSuccess, + ); + const inspect = vi.spyOn( + pythonRuntime, + "inspectLinuxHopperPythonRuntime", + ); + try { + expect( + await host.installPackage("deb", "/tmp/verified-hopper.deb"), + ).toMatchObject({ + status: "failed", + failure: { exitCode: 100, stderr: failed.stderr }, + }); + expect(command).toHaveBeenCalledTimes(failing); + expect(inspect).not.toHaveBeenCalled(); + } finally { + try { + await host.close?.(); + } finally { + inspect.mockRestore(); + command.mockRestore(); + } + } + }, + ); +}); + describe("Linux Hopper package cancellation and disclosure", () => { it.each([ { phase: "before", commands: 0, inspections: 0 }, { phase: "first transaction", commands: 1, inspections: 0 }, - { phase: "query", commands: 1, inspections: 1 }, - { phase: "runtime transaction", commands: 2, inspections: 1 }, - { phase: "verification", commands: 2, inspections: 2 }, + { phase: "archive transaction", commands: debCommands, inspections: 0 }, + { phase: "query", commands: debCommands, inspections: 1 }, + { phase: "runtime transaction", commands: debCommands + 1, inspections: 1 }, + { phase: "verification", commands: debCommands + 1, inspections: 2 }, ])( "honors cancellation at $phase without starting the next stage", async ({ phase, commands, inspections }) => { @@ -224,7 +281,9 @@ describe("Linux Hopper package cancellation and disclosure", () => { commandCount++; if ( (phase === "first transaction" && commandCount === 1) || - (phase === "runtime transaction" && commandCount === 2) + (phase === "archive transaction" && commandCount === debCommands) || + (phase === "runtime transaction" && + commandCount === debCommands + 1) ) controller.abort(cause); return commandSuccess; @@ -267,8 +326,8 @@ describe("Linux Hopper package cancellation and disclosure", () => { it("discloses the conditional privileged library install without returning probe code", () => { const disclosure = linuxHopperInstallDisclosure("deb", false); - expect(disclosure.commands).toHaveLength(2); - expect(disclosure.commands[1]).toBe( + expect(disclosure.commands).toHaveLength(debCommands + 1); + expect(disclosure.commands[debCommands]).toBe( 'pkexec apt-get install -y ""', ); expect(linuxHopperInstallDisclosure("rpm", false).commands).toHaveLength(1); From f4f7c75c669e0c703e508e8d6760b23fe0dc2a01 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:31:12 +0800 Subject: [PATCH 02/37] refactor(artifacts): consolidate selected source lifetimes Use the shared source owner for Mach-O and PyInstaller admission, descriptor and selected-path identity, hashing and cleanup. Retain lipo joining and reader-owned PyInstaller admission, caller cancellation and private PYZ leases. Centralize admission failure projection and remove the superseded PyInstaller wrapper. Preserve physical identity verification before late hash-completion cancellation. Close an owned source before returning failed initial metadata acquisition and retain the exact handle when close fails. A focused regression failed twice before that fix and now passes. Validation: fresh runtime build, 259 focused fixture tests, nine documentation checks and all local static/discovery/duplication gates passed. An earlier check also ran four system-lipo suite cases (three read-only native inspections) before the final source fix; those were excluded from subsequent fixture qualification. --- ...eAdmission.ts => ArtifactFileAdmission.ts} | 21 +- src/artifacts/ArtifactFileSource.ts | 199 ++++--- src/artifacts/MachOSliceArtifactReader.ts | 99 +--- src/artifacts/PyInstallerArtifactReader.ts | 125 +---- .../pyinstaller/PyInstallerPyzStore.ts | 4 +- tests/fixtures/lifecycleAcknowledgement.ts | 28 + .../artifactFileSourceLifecycle.test.ts | 29 +- .../selectedArtifactSourceLifecycle.test.ts | 490 ++++++++++++++++++ 8 files changed, 689 insertions(+), 306 deletions(-) rename src/artifacts/{pyinstaller/PyInstallerFileAdmission.ts => ArtifactFileAdmission.ts} (58%) create mode 100644 tests/fixtures/lifecycleAcknowledgement.ts create mode 100644 tests/process-global/selectedArtifactSourceLifecycle.test.ts diff --git a/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts b/src/artifacts/ArtifactFileAdmission.ts similarity index 58% rename from src/artifacts/pyinstaller/PyInstallerFileAdmission.ts rename to src/artifacts/ArtifactFileAdmission.ts index dc3c503998..fa18c998e0 100644 --- a/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts +++ b/src/artifacts/ArtifactFileAdmission.ts @@ -3,41 +3,44 @@ import { RegularFileAdmissionFailure, RegularFileChangedError, RegularFileOperationCancelled, -} from "../../filesystem/RegularFile.js"; -import { ArtifactReaderFailure } from "../ArtifactReader.js"; +} from "../filesystem/RegularFile.js"; +import { ArtifactReaderFailure } from "./ArtifactReader.js"; -/** Both carrier and private-image admission preserve cancellation and cleanup. */ -export const pyInstallerFileAdmissionFailure = ( +/** Project physical admission failures without taking the caller's descriptor ownership. */ +export const artifactFileAdmissionFailure = ( path: string, + diagnosticLabel: string, cause: unknown, ): ArtifactReaderFailure => { if (cause instanceof ArtifactReaderFailure) return cause; if (cause instanceof RegularFileOperationCancelled) return new ArtifactReaderFailure( "cancelled", - "PyInstaller file admission cancelled", + `${diagnosticLabel} admission cancelled`, { cause: cause.reason }, ); if (cause instanceof RegularFileAdmissionFailure) return ArtifactReaderFailure.withCleanup( - pyInstallerFileAdmissionFailure(path, cause.cause), + artifactFileAdmissionFailure(path, diagnosticLabel, cause.cause), ArtifactReaderFailure.cleanupObservation(cause.cleanupCause, path), ); if (cause instanceof RegularFileChangedError) return new ArtifactReaderFailure( "integrity", - `PyInstaller source changed during admission: ${path}`, + `${diagnosticLabel} source changed during admission: ${path}`, { cause }, ); if (cause instanceof NonRegularFileReadError) return new ArtifactReaderFailure( "format", - `PyInstaller source is not a regular file: ${path}`, + `${diagnosticLabel} source is not a regular file: ${path}`, { cause }, ); return new ArtifactReaderFailure( "io", - "PyInstaller archive could not be opened", + cause instanceof Error + ? cause.message + : `Cannot admit ${diagnosticLabel} source: ${path}`, { cause }, ); }; diff --git a/src/artifacts/ArtifactFileSource.ts b/src/artifacts/ArtifactFileSource.ts index 4eccd7f0dc..4af676b0a8 100644 --- a/src/artifacts/ArtifactFileSource.ts +++ b/src/artifacts/ArtifactFileSource.ts @@ -1,15 +1,15 @@ +import { lstat } from "node:fs/promises"; import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js"; import { - NonRegularFileReadError, RegularFileAdmissionFailure, - RegularFileOperationCancelled, openRegularFile, sameRegularFileState, type StableRegularFileDescriptor, } from "../filesystem/RegularFile.js"; import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; +import { artifactFileAdmissionFailure } from "./ArtifactFileAdmission.js"; import { hashReadable } from "./ArtifactHash.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; @@ -41,41 +41,31 @@ export class ArtifactFileSource { return this.#scope.run(() => this.#verify(signal)); } + /** Verify both the descriptor and the file currently at the caller-selected path. */ + readSelectedPath(signal?: AbortSignal): Promise { + return this.#scope.run(() => this.#verifySelectedPath(signal)); + } + /** Bind verified bytes to the inventory's expected whole-carrier identity. */ bindSha256(expectedSha256: string, signal?: AbortSignal): Promise { - return this.#scope.run(async () => { - const { handle } = await this.#verify(signal); - const observed = - this.#boundSha256 ?? - ( - await hashReadable( - readFileHandleChunks(handle, { - signal, - cancellationFailure: (reason) => - new ArtifactReaderFailure( - "cancelled", - `${this.diagnosticLabel} verification cancelled`, - { cause: reason }, - ), - }), - signal, - ) - ).sha256; - await this.#verify(signal); - if (observed !== expectedSha256) - throw new ArtifactReaderFailure( - "integrity", - `${this.diagnosticLabel} carrier changed after inventory`, - undefined, - artifactIntegrityDetails( - ".", - { algorithm: "sha256", digest: expectedSha256 }, - observed, - false, - ), - ); - this.#boundSha256 = observed; - }); + return this.#scope.run(() => + this.#bind(expectedSha256, signal, () => this.#verify(signal)), + ); + } + + /** Bind carrier bytes while the selected pathname continues to name the same source. */ + bindSelectedPathSha256( + expectedSha256: string, + signal?: AbortSignal, + ): Promise { + return this.#scope.run(() => + this.#bind( + expectedSha256, + signal, + () => this.#verifySelectedPath(signal), + () => this.#verifySelectedPath(), + ), + ); } /** Join admitted operations before closing; failed descriptor ownership stays retained. */ @@ -83,6 +73,46 @@ export class ArtifactFileSource { return this.#scope.close(); } + async #bind( + expectedSha256: string, + signal: AbortSignal | undefined, + verify: () => Promise, + verifyCompletion: () => Promise = verify, + ): Promise { + const { handle } = await verify(); + const observed = + this.#boundSha256 ?? + ( + await hashReadable( + readFileHandleChunks(handle, { + signal, + cancellationFailure: (reason) => + new ArtifactReaderFailure( + "cancelled", + `${this.diagnosticLabel} verification cancelled`, + { cause: reason }, + ), + }), + signal, + ) + ).sha256; + await verifyCompletion(); + assertArtifactReadActive(signal); + if (observed !== expectedSha256) + throw new ArtifactReaderFailure( + "integrity", + `${this.diagnosticLabel} carrier changed after inventory`, + undefined, + artifactIntegrityDetails( + ".", + { algorithm: "sha256", digest: expectedSha256 }, + observed, + false, + ), + ); + this.#boundSha256 = observed; + } + async #verify(signal?: AbortSignal): Promise { assertArtifactReadActive(signal); this.#opening ??= this.#admit(signal); @@ -97,50 +127,65 @@ export class ArtifactFileSource { return source; } - async #admit(signal?: AbortSignal): Promise { - try { - const handle = - this.admitted?.handle ?? - (await openRegularFile(this.path, { symlinks: "reject", signal })); - if (this.admitted === undefined) - this.#owner = new OwnedFileHandle(handle); - const initial = this.admitted?.initial ?? (await handle.stat()); - return { handle, initial }; - } catch (cause: unknown) { - if (cause instanceof RegularFileAdmissionFailure) { - this.#owner = cause.owner; - throw ArtifactReaderFailure.withCleanup( - this.#admissionFailure(cause.cause), - ArtifactReaderFailure.cleanupObservation( - cause.cleanupCause, - this.path, - ), - ); - } - throw this.#admissionFailure(cause); - } + async #verifySelectedPath( + signal?: AbortSignal, + ): Promise { + assertArtifactReadActive(signal); + this.#opening ??= this.#admit(signal); + const source = await this.#opening; + const [observed, selected] = await Promise.all([ + source.handle.stat(), + lstat(this.path), + ]); + if ( + !sameRegularFileState(source.initial, observed) || + !sameRegularFileState(source.initial, selected) + ) + throw new ArtifactReaderFailure( + "integrity", + `${this.diagnosticLabel} source identity changed: ${this.path}`, + ); + assertArtifactReadActive(signal); + return source; } - #admissionFailure(cause: unknown): ArtifactReaderFailure { - if (cause instanceof ArtifactReaderFailure) return cause; - if (cause instanceof RegularFileOperationCancelled) - return new ArtifactReaderFailure( - "cancelled", - `${this.diagnosticLabel} admission cancelled`, - { cause: cause.reason }, + async #admit(signal?: AbortSignal): Promise { + try { + if (this.admitted !== undefined) + return { + handle: this.admitted.handle, + initial: this.admitted.initial, + }; + const handle = await openRegularFile(this.path, { + symlinks: "reject", + signal, + }); + const owner = new OwnedFileHandle(handle); + this.#owner = owner; + try { + return { handle, initial: await handle.stat() }; + } catch (cause: unknown) { + try { + await owner.close(); + this.#owner = undefined; + } catch (cleanupCause: unknown) { + throw new RegularFileAdmissionFailure( + this.path, + cause, + cleanupCause, + owner, + ); + } + throw cause; + } + } catch (cause: unknown) { + if (cause instanceof RegularFileAdmissionFailure) + this.#owner = cause.owner; + throw artifactFileAdmissionFailure( + this.path, + this.diagnosticLabel, + cause, ); - if (cause instanceof NonRegularFileReadError) - return new ArtifactReaderFailure( - "format", - `${this.diagnosticLabel} source is not a regular file: ${this.path}`, - { cause }, - ); - return new ArtifactReaderFailure( - "io", - cause instanceof Error - ? cause.message - : `Cannot admit ${this.diagnosticLabel} source: ${this.path}`, - { cause }, - ); + } } } diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index 791e8a5978..510c0a071d 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -1,5 +1,4 @@ import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; -import { lstat } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; import { Readable } from "node:stream"; @@ -17,7 +16,6 @@ import { machoArchitectureFamily, machoCpuSubtypeBase, } from "../domain/apple/machoCpuIdentity.js"; -import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { readMachoImage, type ReadAt } from "./apple/MachoLoadCommandReader.js"; import { ArtifactReaderFailure, @@ -27,25 +25,17 @@ import { type ArtifactReader, } from "./ArtifactReader.js"; import { streamChunkToBuffer } from "./StreamBytes.js"; -import { - NonRegularFileReadError, - RegularFileAdmissionFailure, - openRegularFile, - sameRegularFileState, - type StableRegularFileDescriptor, -} from "../filesystem/RegularFile.js"; +import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { readArtifactFileChunks } from "./readArtifactFileChunks.js"; +import { ArtifactFileSource } from "./ArtifactFileSource.js"; /** Read-only universal Mach-O slice reader backed by native lipo metadata. */ export class MachOSliceArtifactReader implements ArtifactReader { readonly format = "file" as const; #command: ArtifactCommand | undefined; - #source: StableRegularFileDescriptor | undefined; - #ownedSource: OwnedFileHandle | undefined; - #pendingSource: Promise | undefined; + readonly #source: ArtifactFileSource; readonly #entries = new Map(); readonly #pendingCommands = new Set>(); - readonly #ownsSource: boolean; #closed = false; #closePromise: Promise | undefined; @@ -57,14 +47,13 @@ export class MachOSliceArtifactReader implements ArtifactReader { ), source?: StableRegularFileDescriptor, ) { - this.#source = source; - this.#ownsSource = source === undefined; + this.#source = new ArtifactFileSource(path, "Universal Mach-O", source); } async *entries(signal?: AbortSignal): AsyncIterable { this.#entries.clear(); const source = await this.#ensureSource(); - await this.#verifySource(source); + await this.#source.readSelectedPath(); if (this.#closed) throw new ArtifactReaderFailure( "unavailable", @@ -116,7 +105,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { exit_code: captured.value.exitCode, effects: ["read"], }; - await this.#verifySource(source); + await this.#source.readSelectedPath(); const fileSize = source.initial.size; const parsed = parseLipoOutputResult(captured.value.stdout, this.path); if (!parsed.ok) @@ -128,7 +117,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { const lipo = parsed.value; const { architectures } = lipo; const structural = await readStructuralSlices(source.handle, fileSize); - await this.#verifySource(source); + await this.#source.readSelectedPath(); if (structural.status === "malformed") throw new ArtifactReaderFailure( "integrity", @@ -306,7 +295,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { "Invalid Mach-O slice byte range", ); const source = await this.#ensureSource(); - await this.#verifySource(source); + await this.#source.readSelectedPath(); if ( !Number.isSafeInteger(offset + size) || offset + size > source.initial.size @@ -320,7 +309,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { end: offset + size - 1, ...(signal === undefined ? {} : { signal }), }); - const verifySource = () => this.#verifySource(source); + const verifySource = () => this.#source.readSelectedPath(); let validatingSource = false; const completionFailure = (): ArtifactReaderFailure | undefined => { const coverage = sourceStream.readCoverage(); @@ -456,7 +445,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { ); } try { - await this.#closeSource(); + await this.#source.close(); } catch (cause: unknown) { if (cleanup === undefined) throw cause; throw ArtifactReaderFailure.withCleanup(cause, cleanup); @@ -475,9 +464,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { "unavailable", "Universal Mach-O reader is closed", ); - if (this.#source !== undefined) return this.#source; - this.#pendingSource ??= this.#acquireSource(); - const source = await this.#pendingSource; + const source = await this.#source.read(); if (this.#closed) throw new ArtifactReaderFailure( "unavailable", @@ -485,70 +472,6 @@ export class MachOSliceArtifactReader implements ArtifactReader { ); return source; } - - async #acquireSource(): Promise { - let handle: FileHandle; - try { - handle = await openRegularFile(this.path, { symlinks: "reject" }); - } catch (cause: unknown) { - if (!(cause instanceof RegularFileAdmissionFailure)) throw cause; - this.#ownedSource = cause.owner; - const primary = - cause.cause instanceof NonRegularFileReadError - ? new ArtifactReaderFailure( - "format", - `Universal Mach-O source is not a regular file: ${this.path}`, - { cause: cause.cause }, - ) - : cause.cause; - throw ArtifactReaderFailure.withCleanup( - primary, - ArtifactReaderFailure.cleanupObservation(cause.cleanupCause, this.path), - ); - } - const owner = new OwnedFileHandle(handle); - this.#ownedSource = owner; - try { - const initial = await handle.stat(); - const source = { handle, initial }; - this.#source = source; - return source; - } catch (cause: unknown) { - try { - await owner.close(); - this.#ownedSource = undefined; - } catch (cleanupCause: unknown) { - throw ArtifactReaderFailure.withCleanup( - cause, - ArtifactReaderFailure.cleanupObservation(cleanupCause, this.path), - ); - } - throw cause; - } - } - - async #closeSource(): Promise { - await this.#pendingSource?.catch(() => undefined); - if (!this.#ownsSource || this.#ownedSource === undefined) return; - await this.#ownedSource.close(); - this.#ownedSource = undefined; - this.#source = undefined; - } - - async #verifySource(source: StableRegularFileDescriptor): Promise { - const [opened, currentPath] = await Promise.all([ - source.handle.stat(), - lstat(this.path), - ]); - if ( - !sameRegularFileState(source.initial, opened) || - !sameRegularFileState(source.initial, currentPath) - ) - throw new ArtifactReaderFailure( - "integrity", - `Universal Mach-O source changed during inventory: ${this.path}`, - ); - } } /** Compare lipo's independent ranges with structurally parsed slice identity. */ diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts index 0607105bd7..ea37310538 100644 --- a/src/artifacts/PyInstallerArtifactReader.ts +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -4,7 +4,7 @@ import { setImmediate } from "node:timers/promises"; import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { normalizeArtifactPath } from "./ArtifactPaths.js"; -import { lstat, type FileHandle } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; import type { PyInstallerMetadata } from "../domain/artifactFormatMetadata.js"; import { @@ -12,13 +12,7 @@ import { scriptPythonMagic, } from "./pyinstaller/PyInstallerMetadata.js"; import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; -import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; -import { - RegularFileAdmissionFailure, - openRegularFile, - sameRegularFileState, - type StableRegularFileDescriptor, -} from "../filesystem/RegularFile.js"; +import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { ArtifactReaderFailure, sameArtifactEntry, @@ -27,10 +21,9 @@ import { type ArtifactReaderTraversal, type ArtifactTraversalOptions, } from "./ArtifactReader.js"; -import { abortIfNeeded, hashReadable } from "./ArtifactHash.js"; -import { readArtifactFileChunks } from "./readArtifactFileChunks.js"; +import { abortIfNeeded } from "./ArtifactHash.js"; +import { ArtifactFileSource } from "./ArtifactFileSource.js"; import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; -import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js"; import { parseMzHeaderEvidence } from "../domain/dosMz.js"; import { readFatContainerHeader, @@ -57,7 +50,6 @@ import { } from "./pyinstaller/CArchiveMembers.js"; import { readPyzArchive } from "./pyinstaller/PyzArchive.js"; import type { PyInstallerFailure } from "./pyinstaller/PyInstallerFailure.js"; -import { pyInstallerFileAdmissionFailure } from "./pyinstaller/PyInstallerFileAdmission.js"; import { readPyInstallerBytes } from "./pyinstaller/PyInstallerBytes.js"; const MAX_PARSED_FACT_BYTES = 32 * 1024 * 1024; @@ -117,13 +109,11 @@ interface LoadedArchive { */ export class PyInstallerArtifactReader implements ArtifactReader { readonly format = "pyinstaller" as const; - #source: StableRegularFileDescriptor | undefined; - #opening: Promise | undefined; + readonly #source: ArtifactFileSource; #closed = false; readonly #scope = new ArtifactResourceScope(); readonly #cleanup = new ArtifactResourceScope(); readonly #lifecycle = new AbortController(); - #boundSha256: string | undefined; #loaded: Promise | undefined; #budget = 0; #charged = 0; @@ -134,16 +124,18 @@ export class PyInstallerArtifactReader implements ArtifactReader { readonly #members = new Map(); readonly #openedPyzSizes = new Map(); - constructor( - private readonly path: string, - source?: StableRegularFileDescriptor, - ) { - this.#source = source; + constructor(path: string, source?: StableRegularFileDescriptor) { + this.#source = new ArtifactFileSource(path, "PyInstaller", source); this.#cleanup.retain({ kind: "lease", resource: "PyInstaller private PYZ images", lease: { release: () => this.#pyzStore.close() }, }); + this.#cleanup.retain({ + kind: "lease", + resource: path, + lease: { release: () => this.#source.close() }, + }); } formatMetadata(): PyInstallerMetadata | undefined { @@ -156,31 +148,8 @@ export class PyInstallerArtifactReader implements ArtifactReader { ): Promise { const active = this.#signal(signal); await this.#scope.run(async () => { - const handle = await this.#openHandle(active); - await this.#verifySource(); - const observed = - this.#boundSha256 ?? - ( - await hashReadable( - readArtifactFileChunks(handle, { signal: active }), - active, - ) - ).sha256; - await this.#verifySource(); - abortIfNeeded(active); - if (observed !== expectedSha256) - throw new ArtifactReaderFailure( - "integrity", - "PyInstaller carrier changed after inventory", - undefined, - artifactIntegrityDetails( - ".", - { algorithm: "sha256", digest: expectedSha256 }, - observed, - false, - ), - ); - this.#boundSha256 = observed; + await this.#openHandle(active); + await this.#source.bindSelectedPathSha256(expectedSha256, active); }); } @@ -191,7 +160,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { const active = this.#signal(signal); const loaded = await this.#scope.run(async () => { const value = await this.#load(active); - await this.#verifySource(); + await this.#source.readSelectedPath(); this.#members.clear(); return value; }); @@ -229,7 +198,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { "format", `PyInstaller entry has no readable bytes: ${entry.path}`, ); - await this.#verifySource(); + const { handle } = await this.#source.readSelectedPath(); const knownSize = read.kind === "pyz" ? (read.uncompressedSize ?? @@ -237,11 +206,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { : undefined; if (read.kind === "pyz" && knownSize === undefined) this.#charge(read.header?.length ?? 0); - const parts = await this.#pyzStore.memberSource( - read, - this.#source?.handle, - active, - ); + const parts = await this.#pyzStore.memberSource(read, handle, active); if (active.aborted) { parts.source.destroy(); await finished(parts.source, { cleanup: true }).catch(() => undefined); @@ -273,7 +238,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { complete: async (bytes) => { try { await parts.verify?.(); - await this.#verifySource(); + await this.#source.readSelectedPath(); abortIfNeeded(active); } catch (cause: unknown) { await closeSource(cause); @@ -320,7 +285,6 @@ export class PyInstallerArtifactReader implements ArtifactReader { this.#lifecycle.abort(closing); await this.#scope.close(); await this.#cleanup.close(); - this.#source = undefined; this.#closed = true; this.#loaded = undefined; this.#members.clear(); @@ -345,9 +309,9 @@ export class PyInstallerArtifactReader implements ArtifactReader { } async #parse(signal?: AbortSignal): Promise { - const handle = await this.#openHandle(signal); - await this.#verifySource(); - const size = (await handle.stat()).size; + await this.#openHandle(signal); + const { handle, initial } = await this.#source.readSelectedPath(); + const size = initial.size; const pathDialect = await carrierPathDialect(handle, size, signal); const cookie = await this.#cookie(handle, size, signal); this.#budget = artifactExpansionBudgetBytes(cookie.packageLength); @@ -374,7 +338,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { for (const member of members) if (member.read?.kind === "carchive") this.#charge(member.read.header?.length ?? 0); - await this.#verifySource(); + await this.#source.readSelectedPath(); this.#metadata = cookieMetadata(cookie, toc, encrypted); let opaqueImages = 0; for (const image of images.values()) @@ -400,13 +364,11 @@ export class PyInstallerArtifactReader implements ArtifactReader { "unavailable", "PyInstaller archive is closed", ); - if (this.#source !== undefined) return this.#source.handle; // Admission belongs to the reader. A departing caller cannot cancel a peer // or replace a late descriptor owner; every caller joins the same admission. - this.#opening ??= this.#admit(); let handle: FileHandle; try { - handle = await this.#opening; + ({ handle } = await this.#source.read(this.#lifecycle.signal)); } catch (cause: unknown) { if ( cause instanceof ArtifactReaderFailure && @@ -424,26 +386,6 @@ export class PyInstallerArtifactReader implements ArtifactReader { return handle; } - async #admit(): Promise { - try { - const handle = await openRegularFile(this.path, { - symlinks: "reject", - signal: this.#lifecycle.signal, - }); - this.#retainCarrier(new OwnedFileHandle(handle)); - this.#source = { handle, initial: await handle.stat() }; - return handle; - } catch (cause: unknown) { - if (cause instanceof RegularFileAdmissionFailure) - this.#retainCarrier(cause.owner); - throw pyInstallerFileAdmissionFailure(this.path, cause); - } - } - - #retainCarrier(handle: OwnedFileHandle): void { - this.#cleanup.retain({ kind: "file-handle", resource: this.path, handle }); - } - async #cookie( handle: FileHandle, size: number, @@ -606,27 +548,6 @@ export class PyInstallerArtifactReader implements ArtifactReader { return members; } - async #verifySource(): Promise { - const source = this.#source; - if (source === undefined) - throw new ArtifactReaderFailure( - "unavailable", - "PyInstaller archive is closed", - ); - const [opened, currentPath] = await Promise.all([ - source.handle.stat(), - lstat(this.path), - ]); - if ( - !sameRegularFileState(source.initial, opened) || - !sameRegularFileState(source.initial, currentPath) - ) - throw new ArtifactReaderFailure( - "integrity", - `PyInstaller source changed during reading: ${this.path}`, - ); - } - #reservePathComponents(path: string): void { const remaining = MAX_RETAINED_ENTRY_BYTES - this.#retainedEntryBytes; // Case annotations can carry the path and two prefixes at every component. diff --git a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts index 4b5b053706..30ae081bae 100644 --- a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts +++ b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts @@ -23,7 +23,7 @@ import { } from "./PyInstallerMemberStream.js"; import type { CArchiveEntry } from "./CArchiveLayout.js"; import type { PyInstallerMember } from "./CArchiveMembers.js"; -import { pyInstallerFileAdmissionFailure } from "./PyInstallerFileAdmission.js"; +import { artifactFileAdmissionFailure } from "../ArtifactFileAdmission.js"; import { reconstructedPycHeader, isHeaderlessPythonCode, @@ -242,7 +242,7 @@ export class PyInstallerPyzStore { } catch (cause: unknown) { if (cause instanceof RegularFileAdmissionFailure) this.#retain(path, cause.owner); - throw pyInstallerFileAdmissionFailure(path, cause); + throw artifactFileAdmissionFailure(path, "PyInstaller", cause); } } diff --git a/tests/fixtures/lifecycleAcknowledgement.ts b/tests/fixtures/lifecycleAcknowledgement.ts new file mode 100644 index 0000000000..0789185325 --- /dev/null +++ b/tests/fixtures/lifecycleAcknowledgement.ts @@ -0,0 +1,28 @@ +/** Fail promptly if an operation never reaches its controlled lifecycle boundary. */ +export const awaitAcknowledgement = async ( + acknowledgement: Promise, + operation: Promise, + boundary: string, +): Promise => { + let deadline: ReturnType | undefined; + try { + const observed = await Promise.race([ + acknowledgement.then(() => ({ kind: "acknowledged" }) as const), + operation.then( + (value) => ({ kind: "settled", status: "fulfilled", value }) as const, + (cause: unknown) => + ({ kind: "settled", status: "rejected", cause }) as const, + ), + new Promise<{ readonly kind: "deadline" }>((resolve) => { + deadline = setTimeout(() => resolve({ kind: "deadline" }), 2_000); + }), + ]); + if (observed.kind !== "acknowledged") + throw new Error( + `${boundary} was not acknowledged before ${observed.kind === "deadline" ? "the gate deadline" : "the triggering operation settled"}`, + { cause: observed }, + ); + } finally { + if (deadline !== undefined) clearTimeout(deadline); + } +}; diff --git a/tests/process-global/artifactFileSourceLifecycle.test.ts b/tests/process-global/artifactFileSourceLifecycle.test.ts index 7bf58a85a4..e06e680368 100644 --- a/tests/process-global/artifactFileSourceLifecycle.test.ts +++ b/tests/process-global/artifactFileSourceLifecycle.test.ts @@ -23,6 +23,7 @@ import { nodeSeaElf } from "../../src/artifacts/nodeSea/NodeSea.fixture.js"; import type { StableRegularFileDescriptor } from "../../src/filesystem/RegularFile.js"; import { createDeferred } from "../fixtures/binarySession.js"; import { buildGodotPck } from "../fixtures/godotPck.js"; +import { awaitAcknowledgement } from "../fixtures/lifecycleAcknowledgement.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; interface OpenControl { @@ -101,34 +102,6 @@ const openedHandle = (): FileHandle => { return handle; }; -const awaitAcknowledgement = async ( - acknowledgement: Promise, - operation: Promise, - boundary: string, -): Promise => { - let deadline: ReturnType | undefined; - try { - const observed = await Promise.race([ - acknowledgement.then(() => ({ kind: "acknowledged" }) as const), - operation.then( - (value) => ({ kind: "settled", status: "fulfilled", value }) as const, - (cause: unknown) => - ({ kind: "settled", status: "rejected", cause }) as const, - ), - new Promise<{ readonly kind: "deadline" }>((resolve) => { - deadline = setTimeout(() => resolve({ kind: "deadline" }), 2_000); - }), - ]); - if (observed.kind !== "acknowledged") - throw new Error( - `${boundary} was not acknowledged before ${observed.kind === "deadline" ? "the gate deadline" : "the triggering operation settled"}`, - { cause: observed }, - ); - } finally { - if (deadline !== undefined) clearTimeout(deadline); - } -}; - it.each(carriers)( "$name keeps the caller's expected digest and rechecks identity after binding", async (carrier) => { diff --git a/tests/process-global/selectedArtifactSourceLifecycle.test.ts b/tests/process-global/selectedArtifactSourceLifecycle.test.ts new file mode 100644 index 0000000000..cc76e8ef82 --- /dev/null +++ b/tests/process-global/selectedArtifactSourceLifecycle.test.ts @@ -0,0 +1,490 @@ +import { createHash } from "node:crypto"; +import type { Stats } from "node:fs"; +import { + mkdir, + open, + rename, + writeFile, + type FileHandle, +} from "node:fs/promises"; +import { join } from "node:path"; +import { buffer } from "node:stream/consumers"; +import { setImmediate } from "node:timers/promises"; +import { afterEach, expect, it, vi } from "vitest"; + +import { ArtifactFileSource } from "../../src/artifacts/ArtifactFileSource.js"; +import * as artifactHash from "../../src/artifacts/ArtifactHash.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, +} from "../../src/artifacts/ArtifactReader.js"; +import { MachOSliceArtifactReader } from "../../src/artifacts/MachOSliceArtifactReader.js"; +import { + CPU, + fatImage, + machoImage, +} from "../../src/artifacts/apple/MachoImage.fixture.js"; +import * as regularFile from "../../src/filesystem/RegularFile.js"; +import { createDeferred } from "../fixtures/binarySession.js"; +import { awaitAcknowledgement } from "../fixtures/lifecycleAcknowledgement.js"; +import { NativeFixtureRunner } from "../fixtures/nativeCommands.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +interface OpenControl { + path: string | undefined; + readonly handles: Map Promise>; + onOpen: ((handle: FileHandle) => Promise) | undefined; +} + +const control = vi.hoisted((): OpenControl => ({ + path: undefined, + handles: new Map(), + onOpen: undefined, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + open: async (...args: Parameters) => { + const handle = await original.open(...args); + if (String(args[0]) === control.path) { + control.handles.set(handle, handle.close.bind(handle)); + await control.onOpen?.(handle); + } + return handle; + }, + }; +}); + +afterEach(async () => { + control.onOpen = undefined; + for (const [handle, close] of control.handles) + if (handle.fd >= 0) await close(); + control.handles.clear(); + control.path = undefined; + vi.restoreAllMocks(); +}); + +const machoCarrier = async () => { + const root = await createTestTempDirectory("rea-selected-artifact-source-"); + const selected = join(root, "selected"); + await mkdir(selected); + const path = join(selected, "universal.bin"); + const x86 = Buffer.from(machoImage({ cpu: CPU.x86_64 })); + const arm = Buffer.from(machoImage({ cpu: CPU.arm64 })); + const bytes = Buffer.from( + fatImage([ + { cpu: CPU.x86_64, bytes: x86 }, + { cpu: CPU.arm64, bytes: arm }, + ]), + ); + await writeFile(path, bytes); + control.path = path; + const runner = new NativeFixtureRunner({ + lipo: `architecture x86_64\n cputype 16777223\n cpusubtype 3\n offset 4096\n size ${x86.length}\n align 2^12 (4096)\narchitecture arm64\n cputype 16777228\n cpusubtype 0\n offset 8192\n size ${arm.length}\n align 2^12 (4096)\n`, + }); + return { + path, + root, + selected, + runner, + arm, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +}; + +const entries = async ( + reader: MachOSliceArtifactReader, +): Promise => { + const values: ArtifactEntry[] = []; + for await (const entry of reader.entries()) values.push(entry); + return values; +}; + +const openedHandle = (): FileHandle => { + expect(control.handles.size).toBe(1); + const handle = control.handles.keys().next().value; + if (handle === undefined) + throw new Error("Missing selected carrier descriptor"); + return handle; +}; + +const replaceSelectedDirectory = async ( + fixture: Awaited>, +) => { + await rename(fixture.selected, join(fixture.root, "original")); + await mkdir(fixture.selected); + await writeFile(fixture.path, "replacement selected carrier"); +}; + +const expectDescriptorUnchanged = async ( + handle: FileHandle, + initial: Stats, +) => { + expect(await handle.stat()).toMatchObject({ + dev: initial.dev, + ino: initial.ino, + size: initial.size, + mode: initial.mode, + mtimeMs: initial.mtimeMs, + ctimeMs: initial.ctimeMs, + }); +}; + +it("joins late Mach-O admission before releasing its descriptor and dispatches no lipo after closure", async () => { + const fixture = await machoCarrier(); + const entered = createDeferred(); + const release = createDeferred(); + control.onOpen = async (handle) => { + vi.spyOn(handle, "close"); + entered.resolve(); + await release.promise; + }; + const run = vi.spyOn(fixture.runner, "run"); + const reader = new MachOSliceArtifactReader(fixture.path, {}, fixture.runner); + const operations: Promise[] = []; + try { + const operation = entries(reader); + const reading = operation.catch((cause: unknown) => cause); + operations.push(reading); + await awaitAcknowledgement(entered.promise, operation, "Mach-O admission"); + const handle = openedHandle(); + let closed = false; + const closings = [reader.close(), reader.close()]; + operations.push(...closings); + const closing = Promise.all(closings).then(() => { + closed = true; + }); + operations.push(closing); + await setImmediate(); + expect(closed).toBe(false); + expect(handle.fd).toBeGreaterThanOrEqual(0); + expect(handle.close).not.toHaveBeenCalled(); + release.resolve(); + expect(await reading).toMatchObject({ reason: "unavailable" }); + await closing; + expect(handle.fd).toBe(-1); + expect(handle.close).toHaveBeenCalledOnce(); + await expect(entries(reader)).rejects.toMatchObject({ + reason: "unavailable", + }); + expect(run).not.toHaveBeenCalled(); + expect(control.handles.size).toBe(1); + await reader.close(); + expect(handle.close).toHaveBeenCalledOnce(); + } finally { + release.resolve(); + await Promise.allSettled(operations); + await reader.close(); + } +}); + +it("retains the same Mach-O descriptor through a denied close and concurrent cleanup retry", async () => { + const fixture = await machoCarrier(); + const failure = new Error("controlled Mach-O close denial"); + let blocked = true; + control.onOpen = async (handle) => { + const close = handle.close.bind(handle); + vi.spyOn(handle, "close").mockImplementation(async () => { + if (blocked) throw failure; + await close(); + }); + }; + const reader = new MachOSliceArtifactReader(fixture.path, {}, fixture.runner); + try { + expect((await entries(reader)).map((entry) => entry.path)).toEqual([ + "slices/x86_64", + "slices/arm64", + ]); + const handle = openedHandle(); + expect(await Promise.allSettled([reader.close(), reader.close()])).toEqual([ + { status: "rejected", reason: failure }, + { status: "rejected", reason: failure }, + ]); + expect(handle.fd).toBeGreaterThanOrEqual(0); + expect(handle.close).toHaveBeenCalledOnce(); + blocked = false; + await Promise.all([reader.close(), reader.close()]); + expect(handle.fd).toBe(-1); + expect(handle.close).toHaveBeenCalledTimes(2); + await reader.close(); + expect(handle.close).toHaveBeenCalledTimes(2); + expect(control.handles.size).toBe(1); + } finally { + blocked = false; + await reader.close(); + } +}); + +it("reads admitted Mach-O slice bytes and leaves the borrowed descriptor open after close", async () => { + const fixture = await machoCarrier(); + const handle = await open(fixture.path, "r"); + const close = vi.spyOn(handle, "close"); + let reader: MachOSliceArtifactReader | undefined; + try { + reader = new MachOSliceArtifactReader(fixture.path, {}, fixture.runner, { + handle, + initial: await handle.stat(), + }); + const entry = (await entries(reader)).find( + (value) => value.path === "slices/arm64", + ); + if (entry === undefined) throw new Error("Missing authored ARM64 slice"); + expect(await buffer(await reader.open(entry))).toEqual(fixture.arm); + await reader.close(); + await reader.close(); + expect(close).not.toHaveBeenCalled(); + expect(handle.fd).toBeGreaterThanOrEqual(0); + expect(control.handles.size).toBe(1); + await handle.close(); + expect(close).toHaveBeenCalledOnce(); + expect(handle.fd).toBe(-1); + } finally { + try { + await reader?.close(); + } finally { + if (handle.fd >= 0) await handle.close(); + } + } +}); + +it("rejects a replaced selected pathname without closing the borrowed Mach-O descriptor", async () => { + const fixture = await machoCarrier(); + const handle = await open(fixture.path, "r"); + const close = vi.spyOn(handle, "close"); + const run = vi.spyOn(fixture.runner, "run"); + let reader: MachOSliceArtifactReader | undefined; + try { + const initial = await handle.stat(); + reader = new MachOSliceArtifactReader(fixture.path, {}, fixture.runner, { + handle, + initial, + }); + const entry = (await entries(reader)).find( + (value) => value.path === "slices/arm64", + ); + if (entry === undefined) throw new Error("Missing authored ARM64 slice"); + await replaceSelectedDirectory(fixture); + await expectDescriptorUnchanged(handle, initial); + await expect(reader.open(entry)).rejects.toMatchObject({ + reason: "integrity", + }); + await expect(entries(reader)).rejects.toMatchObject({ + reason: "integrity", + }); + expect(run).toHaveBeenCalledOnce(); + expect(run).toHaveBeenCalledWith( + "lipo", + ["-detailed_info", fixture.path], + {}, + ); + await reader.close(); + expect(close).not.toHaveBeenCalled(); + expect(handle.fd).toBeGreaterThanOrEqual(0); + expect(control.handles.size).toBe(1); + } finally { + try { + await reader?.close(); + } finally { + if (handle.fd >= 0) await handle.close(); + } + } +}); + +it.each([false, true])( + "rejects selected pathname replacement after real hashing even with a late abort (%s)", + async (abort) => { + const fixture = await machoCarrier(); + const completed = createDeferred(); + const release = createDeferred(); + const controller = new AbortController(); + const originalHash = artifactHash.hashReadable; + vi.spyOn(artifactHash, "hashReadable").mockImplementation( + async (...args) => { + const hashed = await originalHash(...args); + expect(hashed.sha256).toBe(fixture.sha256); + completed.resolve(); + await release.promise; + return hashed; + }, + ); + const source = new ArtifactFileSource(fixture.path, "Selected carrier"); + let binding: Promise | undefined; + try { + const operation = source.bindSelectedPathSha256( + fixture.sha256, + controller.signal, + ); + binding = operation.catch((cause: unknown) => cause); + await awaitAcknowledgement( + completed.promise, + operation, + "selected carrier hash", + ); + const handle = openedHandle(); + const initial = await handle.stat(); + await replaceSelectedDirectory(fixture); + await expectDescriptorUnchanged(handle, initial); + if (abort) + controller.abort({ request: "cancel after completed carrier hash" }); + release.resolve(); + const failure = await binding; + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected a selected-path identity failure"); + expect(failure.reason).toBe("integrity"); + expect(failure.message).toBe( + `Selected carrier source identity changed: ${fixture.path}`, + ); + expect(failure.cancellation).toBeUndefined(); + await source.close(); + expect(handle.fd).toBe(-1); + } finally { + release.resolve(); + await binding; + await source.close(); + } + }, +); + +it.each([false, true])( + "preserves typed admission identity failure and its exact cleanup owner (closeFails=%s)", + async (closeFails) => { + const fixture = await machoCarrier(); + const changed = new regularFile.RegularFileChangedError(fixture.path); + const cleanupFailure = new Error( + "controlled changed-admission close denial", + ); + let blocked = closeFails; + control.onOpen = async (handle) => { + vi.spyOn(handle, "stat").mockRejectedValueOnce(changed); + const close = handle.close.bind(handle); + vi.spyOn(handle, "close").mockImplementation(async () => { + if (blocked) throw cleanupFailure; + await close(); + }); + }; + const run = vi.spyOn(fixture.runner, "run"); + const reader = new MachOSliceArtifactReader( + fixture.path, + {}, + fixture.runner, + ); + try { + const failure = await entries(reader).catch((cause: unknown) => cause); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected a typed admission identity failure"); + expect(failure.reason).toBe("integrity"); + const primary = closeFails ? failure.cause : failure; + if (!(primary instanceof ArtifactReaderFailure)) + throw new Error("Missing the primary admission identity failure"); + expect(primary.cause).toBe(changed); + const handle = openedHandle(); + if (closeFails) { + expect(failure.cleanup).toEqual({ + reason: cleanupFailure.message, + resources: [fixture.path], + }); + expect(handle.fd).toBeGreaterThanOrEqual(0); + await expect(reader.close()).rejects.toBe(cleanupFailure); + expect(handle.close).toHaveBeenCalledTimes(2); + blocked = false; + await reader.close(); + expect(handle.close).toHaveBeenCalledTimes(3); + } else { + expect(failure.cleanup).toBeUndefined(); + await reader.close(); + expect(handle.close).toHaveBeenCalledOnce(); + } + expect(handle.fd).toBe(-1); + expect(run).not.toHaveBeenCalled(); + expect(control.handles.size).toBe(1); + } finally { + blocked = false; + await reader.close(); + } + }, +); + +it.each([false, true])( + "closes an owned Mach-O source before rejecting its initial stat failure (invalidatingCloseFailure=%s)", + async (invalidatingCloseFailure) => { + const fixture = await machoCarrier(); + const statFailure = Object.assign( + new Error("post-admission initial stat failed"), + { + code: "EIO", + errno: -5, + syscall: "fstat", + }, + ); + const cleanupFailure = new Error( + "close rejected after descriptor invalidation", + ); + const originalAdmission = regularFile.openRegularFile; + vi.spyOn(regularFile, "openRegularFile").mockImplementation( + async (...args) => { + const handle = await originalAdmission(...args); + if (args[0] === fixture.path) { + // Admission has completed; the owner's next stat establishes its initial facts. + vi.spyOn(handle, "stat").mockRejectedValueOnce(statFailure); + const close = handle.close.bind(handle); + vi.spyOn(handle, "close").mockImplementation(async () => { + await close(); + if (invalidatingCloseFailure) throw cleanupFailure; + }); + } + return handle; + }, + ); + const run = vi.spyOn(fixture.runner, "run"); + const reader = new MachOSliceArtifactReader( + fixture.path, + {}, + fixture.runner, + ); + try { + const failure = await entries(reader).catch((cause: unknown) => { + const handle = openedHandle(); + expect(handle.close).toHaveBeenCalledOnce(); + expect(handle.fd).toBe(-1); + return cause; + }); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected the primary initial-stat failure"); + expect(failure.reason).toBe("io"); + const primary = invalidatingCloseFailure ? failure.cause : failure; + if (!(primary instanceof ArtifactReaderFailure)) + throw new Error("Missing initial-stat primary failure"); + expect(primary.cause).toBe(statFailure); + const handle = openedHandle(); + if (invalidatingCloseFailure) { + expect(failure.cleanup).toEqual({ + reason: cleanupFailure.message, + resources: [fixture.path], + }); + expect( + await Promise.allSettled([reader.close(), reader.close()]), + ).toEqual([ + { status: "rejected", reason: cleanupFailure }, + { status: "rejected", reason: cleanupFailure }, + ]); + await expect(reader.close()).rejects.toBe(cleanupFailure); + } else { + expect(failure.cleanup).toBeUndefined(); + await Promise.all([reader.close(), reader.close()]); + await reader.close(); + } + expect(handle.close).toHaveBeenCalledOnce(); + expect(handle.fd).toBe(-1); + expect(run).not.toHaveBeenCalled(); + expect(control.handles.size).toBe(1); + } finally { + if (invalidatingCloseFailure) + await reader.close().catch((cause: unknown) => { + expect(cause).toBe(cleanupFailure); + }); + else await reader.close(); + } + }, +); From 2acdd54130c7a07c78168ff1b2d8f98b598319ac Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:40:47 +0800 Subject: [PATCH 03/37] test(setup): simplify prewarm fixture ownership Flatten the prewarm cases so suite grouping does not exceed the function-size gate. Let createTestTempDirectory own cleanup instead of deleting the same roots from a second afterEach owner. Preserve all registration and warning assertions. Validation: all four focused filesystem cases passed using temporary client configurations and injected prewarm results; no package runner or provider was launched. --- .../setupRegistrationPrewarm.test.ts | 397 +++++++++--------- 1 file changed, 192 insertions(+), 205 deletions(-) diff --git a/tests/boundary/filesystem/setupRegistrationPrewarm.test.ts b/tests/boundary/filesystem/setupRegistrationPrewarm.test.ts index 63fa10e838..37b1300725 100644 --- a/tests/boundary/filesystem/setupRegistrationPrewarm.test.ts +++ b/tests/boundary/filesystem/setupRegistrationPrewarm.test.ts @@ -1,6 +1,6 @@ -import { mkdir, rm } from "node:fs/promises"; +import { mkdir } from "node:fs/promises"; import { dirname } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { expect, it } from "vitest"; import { createDoctorHostFixture } from "../../../src/application/Doctor.fixture.js"; import { systemSetupHost } from "../../../src/application/SetupHost.js"; @@ -11,13 +11,6 @@ import { supportedClients } from "../../../src/application/SupportedClients.js"; import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -const roots: string[] = []; -afterEach(async () => { - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); -}); - /** * A package-runner launch environment that also satisfies the fixture's * Hopper launcher probe, so setup reports readiness instead of proposing a @@ -29,205 +22,199 @@ const packageRunnerEnvironment = { HOPPER_LAUNCHER_PATH: "/fixture/Hopper", } as const; -describe("setup registration package-runner prewarm", () => { - it("materializes the pinned runner cache once after configuring clients", async () => { - const root = await createTestTempDirectory("rea-setup-prewarm-"); - roots.push(root); - const client = supportedClients(root, "darwin", {}).find( - (candidate) => candidate.name === "cursor", - ); - if (client === undefined) throw new Error("Cursor registration is missing"); - await mkdir(dirname(client.configPath), { recursive: true }); - const doctor = createDoctorHostFixture({ - clientRegistrations: () => - readClientRegistrationStatuses(root, undefined, { - platform: "darwin", - environment: {}, - }), - providerInspections: () => - Promise.resolve([ - { - id: "hopper", - configured: true, - available: false, - providerVersion: "6", - registrationEnvironment: {}, - checks: [ - { - name: "hopper-license", - ok: false, - code: "unavailable", - detail: "Runtime license is unavailable", - classification: "missing_analysis_engine", - remediation: null, - }, - ], - }, - ]), - }); - let prewarmCalls = 0; - const prewarmResult: SetupRegistrationPrewarm = { status: "warmed" }; - const host = { - ...systemSetupHost(doctor, packageRunnerEnvironment), - // The host reports the developer machine's Node; pin a supported one - // so readiness does not depend on the local runtime. - nodeVersion: "24.22.1", - detectedClients: () => Promise.resolve([client]), - supportedClients: () => Promise.resolve([client]), - prewarmRegistrationRunner: () => { - prewarmCalls += 1; - return Promise.resolve(prewarmResult); - }, - }; - const result = await runSetup( - { - ...options(true), - clientIds: ["cursor"], - installSkill: false, - proposeHopper: false, - readinessScope: { clients: [], providers: [], skill: false }, - }, - host, - ); - expect(result.status).toBe("ready"); - expect(result.clients.cursor).toMatchObject({ status: "configured" }); - expect(prewarmCalls).toBe(1); - expect(result.prewarm).toEqual(prewarmResult); +it("setup prewarms the pinned runner cache once after configuring clients", async () => { + const root = await createTestTempDirectory("rea-setup-prewarm-"); + const client = supportedClients(root, "darwin", {}).find( + (candidate) => candidate.name === "cursor", + ); + if (client === undefined) throw new Error("Cursor registration is missing"); + await mkdir(dirname(client.configPath), { recursive: true }); + const doctor = createDoctorHostFixture({ + clientRegistrations: () => + readClientRegistrationStatuses(root, undefined, { + platform: "darwin", + environment: {}, + }), + providerInspections: () => + Promise.resolve([ + { + id: "hopper", + configured: true, + available: false, + providerVersion: "6", + registrationEnvironment: {}, + checks: [ + { + name: "hopper-license", + ok: false, + code: "unavailable", + detail: "Runtime license is unavailable", + classification: "missing_analysis_engine", + remediation: null, + }, + ], + }, + ]), }); + let prewarmCalls = 0; + const prewarmResult: SetupRegistrationPrewarm = { status: "warmed" }; + const host = { + ...systemSetupHost(doctor, packageRunnerEnvironment), + // The host reports the developer machine's Node; pin a supported one + // so readiness does not depend on the local runtime. + nodeVersion: "24.22.1", + detectedClients: () => Promise.resolve([client]), + supportedClients: () => Promise.resolve([client]), + prewarmRegistrationRunner: () => { + prewarmCalls += 1; + return Promise.resolve(prewarmResult); + }, + }; + const result = await runSetup( + { + ...options(true), + clientIds: ["cursor"], + installSkill: false, + proposeHopper: false, + readinessScope: { clients: [], providers: [], skill: false }, + }, + host, + ); + expect(result.status).toBe("ready"); + expect(result.clients.cursor).toMatchObject({ status: "configured" }); + expect(prewarmCalls).toBe(1); + expect(result.prewarm).toEqual(prewarmResult); +}); - it("keeps setup ready when the prewarm fails, with a warning progress event", async () => { - const root = await createTestTempDirectory("rea-setup-prewarm-fail-"); - roots.push(root); - const client = supportedClients(root, "darwin", {}).find( - (candidate) => candidate.name === "cursor", - ); - if (client === undefined) throw new Error("Cursor registration is missing"); - await mkdir(dirname(client.configPath), { recursive: true }); - const doctor = createDoctorHostFixture({ - clientRegistrations: () => - readClientRegistrationStatuses(root, undefined, { - platform: "darwin", - environment: {}, - }), - providerInspections: () => Promise.resolve([]), - }); - const progress: { state: string; detail?: string }[] = []; - const host = { - ...systemSetupHost(doctor, packageRunnerEnvironment), - // The host reports the developer machine's Node; pin a supported one - // so readiness does not depend on the local runtime. - nodeVersion: "24.22.1", - detectedClients: () => Promise.resolve([client]), - supportedClients: () => Promise.resolve([client]), - prewarmRegistrationRunner: () => - Promise.resolve({ - status: "failed" as const, - reason: "The pinned package-runner entry could not be materialized", - }), - }; - const result = await runSetup( - { - ...options(true), - clientIds: ["cursor"], - installSkill: false, - proposeHopper: false, - readinessScope: { clients: [], providers: [], skill: false }, - onProgress: (event) => - progress.push({ - state: event.state, - ...(event.detail === undefined ? {} : { detail: event.detail }), - }), - }, - host, - ); - expect(result.status).toBe("ready"); - expect(result.prewarm).toMatchObject({ status: "failed" }); - expect(progress).toContainEqual({ - state: "warning", - detail: "The pinned package-runner entry could not be materialized", - }); +it("keeps setup ready when registration prewarm fails, with a warning progress event", async () => { + const root = await createTestTempDirectory("rea-setup-prewarm-fail-"); + const client = supportedClients(root, "darwin", {}).find( + (candidate) => candidate.name === "cursor", + ); + if (client === undefined) throw new Error("Cursor registration is missing"); + await mkdir(dirname(client.configPath), { recursive: true }); + const doctor = createDoctorHostFixture({ + clientRegistrations: () => + readClientRegistrationStatuses(root, undefined, { + platform: "darwin", + environment: {}, + }), + providerInspections: () => Promise.resolve([]), }); - - it("omits the prewarm summary when the host offers no capability", async () => { - const root = await createTestTempDirectory("rea-setup-prewarm-absent-"); - roots.push(root); - const client = supportedClients(root, "darwin", {}).find( - (candidate) => candidate.name === "cursor", - ); - if (client === undefined) throw new Error("Cursor registration is missing"); - await mkdir(dirname(client.configPath), { recursive: true }); - const doctor = createDoctorHostFixture({ - clientRegistrations: () => - readClientRegistrationStatuses(root, undefined, { - platform: "darwin", - environment: {}, + const progress: { state: string; detail?: string }[] = []; + const host = { + ...systemSetupHost(doctor, packageRunnerEnvironment), + // The host reports the developer machine's Node; pin a supported one + // so readiness does not depend on the local runtime. + nodeVersion: "24.22.1", + detectedClients: () => Promise.resolve([client]), + supportedClients: () => Promise.resolve([client]), + prewarmRegistrationRunner: () => + Promise.resolve({ + status: "failed" as const, + reason: "The pinned package-runner entry could not be materialized", + }), + }; + const result = await runSetup( + { + ...options(true), + clientIds: ["cursor"], + installSkill: false, + proposeHopper: false, + readinessScope: { clients: [], providers: [], skill: false }, + onProgress: (event) => + progress.push({ + state: event.state, + ...(event.detail === undefined ? {} : { detail: event.detail }), }), - providerInspections: () => Promise.resolve([]), - }); - const { prewarmRegistrationRunner: absent, ...hostWithoutPrewarm } = - systemSetupHost(doctor, packageRunnerEnvironment); - void absent; - const host = { - ...hostWithoutPrewarm, - // The host reports the developer machine's Node; pin a supported one - // so readiness does not depend on the local runtime. - nodeVersion: "24.22.1", - detectedClients: () => Promise.resolve([client]), - supportedClients: () => Promise.resolve([client]), - }; - const result = await runSetup( - { - ...options(true), - clientIds: ["cursor"], - installSkill: false, - proposeHopper: false, - readinessScope: { clients: [], providers: [], skill: false }, - }, - host, - ); - expect(result.status).toBe("ready"); - expect(result.prewarm).toBeUndefined(); - }); - - it("skips the prewarm when no client registration was applied", async () => { - const root = await createTestTempDirectory("rea-setup-prewarm-skip-"); - roots.push(root); - const client = supportedClients(root, "darwin", {}).find( - (candidate) => candidate.name === "cursor", - ); - if (client === undefined) throw new Error("Cursor registration is missing"); - await mkdir(dirname(client.configPath), { recursive: true }); - const doctor = createDoctorHostFixture({ - clientRegistrations: () => - readClientRegistrationStatuses(root, undefined, { - platform: "darwin", - environment: {}, - }), - providerInspections: () => Promise.resolve([]), - }); - let prewarmCalls = 0; - const host = { - ...systemSetupHost(doctor, packageRunnerEnvironment), - nodeVersion: "24.22.1", - detectedClients: () => Promise.resolve([client]), - supportedClients: () => Promise.resolve([client]), - prewarmRegistrationRunner: () => { - prewarmCalls += 1; - return Promise.resolve({ status: "warmed" as const }); - }, - }; - const result = await runSetup( - { - ...options(true), - clientIds: [], - installSkill: false, - proposeHopper: false, - readinessScope: { clients: [], providers: [], skill: false }, - }, - host, - ); - expect(result.status).toBe("ready"); - expect(prewarmCalls).toBe(0); - expect(result.prewarm).toBeUndefined(); + }, + host, + ); + expect(result.status).toBe("ready"); + expect(result.prewarm).toMatchObject({ status: "failed" }); + expect(progress).toContainEqual({ + state: "warning", + detail: "The pinned package-runner entry could not be materialized", }); }); + +it("setup omits the prewarm summary when the host offers no capability", async () => { + const root = await createTestTempDirectory("rea-setup-prewarm-absent-"); + const client = supportedClients(root, "darwin", {}).find( + (candidate) => candidate.name === "cursor", + ); + if (client === undefined) throw new Error("Cursor registration is missing"); + await mkdir(dirname(client.configPath), { recursive: true }); + const doctor = createDoctorHostFixture({ + clientRegistrations: () => + readClientRegistrationStatuses(root, undefined, { + platform: "darwin", + environment: {}, + }), + providerInspections: () => Promise.resolve([]), + }); + const { prewarmRegistrationRunner: absent, ...hostWithoutPrewarm } = + systemSetupHost(doctor, packageRunnerEnvironment); + void absent; + const host = { + ...hostWithoutPrewarm, + // The host reports the developer machine's Node; pin a supported one + // so readiness does not depend on the local runtime. + nodeVersion: "24.22.1", + detectedClients: () => Promise.resolve([client]), + supportedClients: () => Promise.resolve([client]), + }; + const result = await runSetup( + { + ...options(true), + clientIds: ["cursor"], + installSkill: false, + proposeHopper: false, + readinessScope: { clients: [], providers: [], skill: false }, + }, + host, + ); + expect(result.status).toBe("ready"); + expect(result.prewarm).toBeUndefined(); +}); + +it("setup skips prewarm when no client registration was applied", async () => { + const root = await createTestTempDirectory("rea-setup-prewarm-skip-"); + const client = supportedClients(root, "darwin", {}).find( + (candidate) => candidate.name === "cursor", + ); + if (client === undefined) throw new Error("Cursor registration is missing"); + await mkdir(dirname(client.configPath), { recursive: true }); + const doctor = createDoctorHostFixture({ + clientRegistrations: () => + readClientRegistrationStatuses(root, undefined, { + platform: "darwin", + environment: {}, + }), + providerInspections: () => Promise.resolve([]), + }); + let prewarmCalls = 0; + const host = { + ...systemSetupHost(doctor, packageRunnerEnvironment), + nodeVersion: "24.22.1", + detectedClients: () => Promise.resolve([client]), + supportedClients: () => Promise.resolve([client]), + prewarmRegistrationRunner: () => { + prewarmCalls += 1; + return Promise.resolve({ status: "warmed" as const }); + }, + }; + const result = await runSetup( + { + ...options(true), + clientIds: [], + installSkill: false, + proposeHopper: false, + readinessScope: { clients: [], providers: [], skill: false }, + }, + host, + ); + expect(result.status).toBe("ready"); + expect(prewarmCalls).toBe(0); + expect(result.prewarm).toBeUndefined(); +}); From c7ae27807f0f952c8fa23a4967fd7a2d4d39d783 Mon Sep 17 00:00:00 2001 From: Wenyu Chiou <162016108+WenyuChiou@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:57:02 -0400 Subject: [PATCH 04/37] fix(electron): retain scoped IPC registration provenance (#7031) * fix(electron): retain scoped IPC registration provenance * fix(electron): defer mutation analysis until scoped receiver admission --- .../javascript/ElectronBoundaryAnalysis.ts | 8 +- .../javascript/ElectronBoundaryGraphIpc.ts | 58 +++++++-- .../javascript/electronStaticAnalysisIpc.ts | 49 ++++++- .../electronStaticAnalysisScopedIpc.test.ts | 120 ++++++++++++++++++ .../javascript/electronStaticAnalysisTypes.ts | 7 + .../electronStaticReceiverAdmission.test.ts | 80 ++++++++++++ .../javascript/electronStaticRenderer.ts | 95 +++++++++++--- .../javascript/javascriptStaticAnalysis.ts | 7 +- .../javascriptStaticAnalysisState.ts | 7 + .../JavaScriptAnalysisWorkerSchemas.test.ts | 29 +++++ .../JavaScriptAnalysisWorkerSchemas.ts | 8 ++ tests/boundary/cli/electronScopedIpc.test.ts | 114 +++++++++++++++++ 12 files changed, 546 insertions(+), 36 deletions(-) create mode 100644 src/domain/javascript/electronStaticAnalysisScopedIpc.test.ts create mode 100644 src/domain/javascript/electronStaticReceiverAdmission.test.ts create mode 100644 tests/boundary/cli/electronScopedIpc.test.ts diff --git a/src/application/javascript/ElectronBoundaryAnalysis.ts b/src/application/javascript/ElectronBoundaryAnalysis.ts index 7b7b5c4bc6..d5d6ab5962 100644 --- a/src/application/javascript/ElectronBoundaryAnalysis.ts +++ b/src/application/javascript/ElectronBoundaryAnalysis.ts @@ -54,16 +54,18 @@ export const classifyElectronIpcPairings = ( ? [] : handlers.filter( (handler) => - handler.finding.channel === transmission.finding.channel && + (handler.finding.channel === transmission.finding.channel || + (handler.finding.receiver !== undefined && + handler.finding.channel === null)) && compatibleIpcModes(transmission.finding, handler.finding), ); return { transmission, candidates, status: - candidates.length === 1 + candidates.length === 1 && candidates[0]?.finding.receiver === undefined ? "paired" - : candidates.length > 1 + : candidates.length > 0 ? "ambiguous" : "unpaired", }; diff --git a/src/application/javascript/ElectronBoundaryGraphIpc.ts b/src/application/javascript/ElectronBoundaryGraphIpc.ts index 3cc383914d..23b694ffdb 100644 --- a/src/application/javascript/ElectronBoundaryGraphIpc.ts +++ b/src/application/javascript/ElectronBoundaryGraphIpc.ts @@ -119,10 +119,22 @@ const addChannelNode = ( coverage: JavaScriptArtifactGraphCoverage, ): ApplicationNode => { const { finding, file } = record; - const identityKey = + const channelKey = finding.channel === null ? `dynamic:${record.key}` : `literal:${finding.channel}`; + const identityKey = + finding.receiver === undefined + ? channelKey + : JSON.stringify([ + file.path, + file.sha256, + finding.module_runtime ?? null, + finding.module_key, + finding.receiver.kind, + finding.receiver.owner_location, + channelKey, + ]); return context.accumulator.addNode({ kind: "ipc-channel", identity: @@ -141,6 +153,9 @@ const addChannelNode = ( { label: finding.channel ?? finding.channel_expression, properties: { + ...(finding.receiver === undefined + ? {} + : { receiver: finding.receiver }), side: finding.side, operation: finding.operation, mode: finding.mode, @@ -159,16 +174,23 @@ const addChannelNode = ( range: finding.location, operation: "observe-ipc-channel-expression", coverage, - limitations: - finding.resolution_basis === "wrapper-substitution" - ? [ + limitations: [ + ...(finding.receiver === undefined + ? [] + : [ + "The receiver is a source-level allocation, not a runtime frame identity; registration and dispatch remain unverified.", + ]), + ...(finding.resolution_basis !== "wrapper-substitution" + ? [] + : [ "The channel is inferred through lexical wrapper substitution; runtime registration and dispatch remain unverified.", - ] - : finding.channel === null - ? [ - "The channel expression is dynamic; its runtime value and pairings remain unknown.", - ] - : [], + ]), + ...(finding.channel !== null + ? [] + : [ + "The channel expression is dynamic; its runtime value and pairings remain unknown.", + ]), + ], }), }, ], @@ -179,17 +201,31 @@ const addHandlerNode = (input: IpcHandlerInput): ApplicationNode => { const { context, record, source, channel, coverage } = input; const { finding, file } = record; const handlerRange = finding.handler_location ?? finding.location; + const handlerKey = `${finding.side}:${finding.operation}:${finding.channel ?? finding.channel_expression}:${javascriptSourceRangeKey(handlerRange)}`; + const identityKey = + finding.receiver === undefined + ? handlerKey + : JSON.stringify([ + file.path, + finding.module_runtime ?? null, + finding.module_key, + finding.receiver, + handlerKey, + ]); const handler = context.accumulator.addNode({ kind: "ipc-handler", identity: artifactLocalIdentity( file.sha256, "electron-ipc-handler", - `${finding.side}:${finding.operation}:${finding.channel ?? finding.channel_expression}:${javascriptSourceRangeKey(handlerRange)}`, + identityKey, ), observations: [ { label: finding.channel, properties: { + ...(finding.receiver === undefined + ? {} + : { receiver: finding.receiver }), side: finding.side, operation: finding.operation, mode: finding.mode, diff --git a/src/domain/javascript/electronStaticAnalysisIpc.ts b/src/domain/javascript/electronStaticAnalysisIpc.ts index 5cebf0cc92..43bcdbc383 100644 --- a/src/domain/javascript/electronStaticAnalysisIpc.ts +++ b/src/domain/javascript/electronStaticAnalysisIpc.ts @@ -3,6 +3,7 @@ import * as t from "@babel/types"; import type { ElectronIpcDescriptor, ElectronIpcFinding, + ElectronIpcReceiver, ElectronSenderValidationFinding, } from "./electronStaticAnalysisTypes.js"; import { @@ -37,7 +38,10 @@ const inspectIpcCall = ( node: t.CallExpression | t.OptionalCallExpression, context: JavaScriptFindingContext, ): void => { - const descriptor = ipcDescriptor(electronCalleePath(node.callee, context)); + const scoped = scopedIpcDescriptor(node, context); + const descriptor = + scoped?.descriptor ?? + ipcDescriptor(electronCalleePath(node.callee, context)); if (descriptor === undefined) return; const channelNode = argumentNode(node.arguments[0]); const channelValue = electronStaticValue(context.source, channelNode); @@ -88,6 +92,7 @@ const inspectIpcCall = ( context.accumulator.unknownFindings += 1; const finding: ElectronIpcFinding = { ...findingState, + ...(scoped === undefined ? {} : { receiver: scoped.receiver }), ...channelReference, module_key: null, location: range(node), @@ -100,6 +105,48 @@ const inspectIpcCall = ( }); }; +const scopedIpcDescriptor = ( + node: t.CallExpression | t.OptionalCallExpression, + context: JavaScriptFindingContext, +): + | { + readonly descriptor: ElectronIpcDescriptor; + readonly receiver: ElectronIpcReceiver; + } + | undefined => { + if ( + !t.isMemberExpression(node.callee) && + !t.isOptionalMemberExpression(node.callee) + ) + return undefined; + const operation = semanticStaticPropertyName( + node.callee.property, + node.callee.computed, + ); + const descriptor = ipcDescriptor(["ipcMain", operation]); + if (descriptor === undefined) return undefined; + const receiver = context.browserWindowReceiver?.(node.callee.object, node, [ + operation, + ]); + if (receiver === undefined) return undefined; + const path = receiver.path; + const kind = + path.length === 3 && path[0] === "webContents" && path[1] === "ipc" + ? "web-contents" + : path.length === 4 && + path[0] === "webContents" && + path[1] === "mainFrame" && + path[2] === "ipc" + ? "main-frame" + : undefined; + return kind === undefined + ? undefined + : { + descriptor, + receiver: { kind, owner_location: range(receiver.owner) }, + }; +}; + const inspectValidationComparison = ( node: t.BinaryExpression, context: JavaScriptFindingContext, diff --git a/src/domain/javascript/electronStaticAnalysisScopedIpc.test.ts b/src/domain/javascript/electronStaticAnalysisScopedIpc.test.ts new file mode 100644 index 0000000000..0cefe731c2 --- /dev/null +++ b/src/domain/javascript/electronStaticAnalysisScopedIpc.test.ts @@ -0,0 +1,120 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it.each([ + ["webContents", "window.webContents.ipc"], + ["main frame", "window.webContents.mainFrame.ipc"], +] as const)( + "retains a proven %s IPC handler beside the global control", + (_, receiver) => { + const analysis = analyzeJavaScriptStaticSource(` + const { BrowserWindow, ipcMain } = require("electron"); + const window = new BrowserWindow(); + ipcMain.handle("global-control", () => "global"); + ${receiver}.handle("scoped-channel", () => "scoped"); + `); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.browser_windows).toHaveLength(1); + expect(analysis.electron.ipc).toContainEqual( + expect.objectContaining({ + side: "main", + operation: "handle", + channel: "global-control", + handler_kind: "inline-function", + }), + ); + expect(analysis.electron.ipc).toContainEqual( + expect.objectContaining({ + side: "main", + operation: "handle", + channel: "scoped-channel", + handler_kind: "inline-function", + }), + ); + }, +); + +it("does not label lookalike or shadowed receivers as Electron IPC", () => { + const analysis = analyzeJavaScriptStaticSource(` + const { BrowserWindow, ipcMain } = require("electron"); + ipcMain.handle("global-control", () => "global"); + const fake = { webContents: { ipc: { handle() {} } } }; + fake.webContents.ipc.handle("fake", () => {}); + function shadow(BrowserWindow) { + const window = new BrowserWindow(); + window.webContents.ipc.handle("shadowed", () => {}); + } + `); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.ipc.map(({ channel }) => channel)).toEqual([ + "global-control", + ]); +}); + +it.each([ + [ + 'const contents = window.webContents; contents.ipc.handle("same", () => {});', + "web-contents", + ], + [ + 'const frame = window.webContents.mainFrame; frame.ipc.handle("same", () => {});', + "main-frame", + ], + [ + 'const contents = window.webContents; const frame = contents.mainFrame; const ipc = frame.ipc; ipc.handle("same", () => {});', + "main-frame", + ], +])("retains a proven scoped alias: %s", (call, kind) => { + const findings = + analyzeJavaScriptStaticSource(`const { BrowserWindow } = require("electron"); + const window = new BrowserWindow(); ${call}`).electron.ipc; + expect(findings).toHaveLength(1); + expect(findings[0]?.receiver).toEqual({ + kind, + owner_location: { + start: { line: 2, column: 19 }, + end: { line: 2, column: 38 }, + }, + }); +}); + +it("keeps the shared owner's conservative omission after an aliased receiver call", () => { + const findings = + analyzeJavaScriptStaticSource(`const { BrowserWindow } = require("electron"); + const window = new BrowserWindow(); + const ipc = window.webContents.mainFrame.ipc; + ipc.handle("first", () => {}); + window.webContents.mainFrame.ipc.on("after-escape", () => {});`).electron + .ipc; + expect(findings.map(({ channel }) => channel)).toEqual(["first"]); +}); + +it.each([ + "window.webContents = replacement;", + "window.webContents.ipc = replacement;", + "window.webContents.ipc.handle = replacement;", + "const alias = window; alias.webContents.ipc.handle = replacement;", + "const contents = window.webContents; contents.ipc = replacement;", + "BrowserWindow.prototype.webContents = replacement;", +])("omits a mutated receiver: %s", (mutation) => { + const analysis = analyzeJavaScriptStaticSource(` + const { BrowserWindow } = require("electron"); + const window = new BrowserWindow(); + ${mutation} + window.webContents.ipc.handle("changed", () => {}); + `); + expect(analysis.parse_error_count).toBe(0); + expect(analysis.electron.ipc).toEqual([]); +}); + +it.each(["webContents.ipc", "webContents.mainFrame.ipc"])( + "does not split one dotted member key into receiver segments: %s", + (key) => { + expect( + analyzeJavaScriptStaticSource(`const {BrowserWindow} = require("electron"); + const window = new BrowserWindow(); window[${JSON.stringify(key)}].handle("fake", () => {});`) + .electron.ipc, + ).toEqual([]); + }, +); diff --git a/src/domain/javascript/electronStaticAnalysisTypes.ts b/src/domain/javascript/electronStaticAnalysisTypes.ts index b0e4aca045..7cba97065b 100644 --- a/src/domain/javascript/electronStaticAnalysisTypes.ts +++ b/src/domain/javascript/electronStaticAnalysisTypes.ts @@ -114,7 +114,14 @@ type ElectronIpcHandler = readonly handler_location: JavaScriptSourceRange; }; +export interface ElectronIpcReceiver { + readonly kind: "web-contents" | "main-frame"; + /** Constructor syntax allocation, never a runtime frame identifier. */ + readonly owner_location: JavaScriptSourceRange; +} + interface ElectronIpcFindingContext { + readonly receiver?: ElectronIpcReceiver; /** Absent on direct syntax observations; wrappers require inferred evidence. */ readonly resolution_basis?: "wrapper-substitution"; readonly module_key: string | null; diff --git a/src/domain/javascript/electronStaticReceiverAdmission.test.ts b/src/domain/javascript/electronStaticReceiverAdmission.test.ts new file mode 100644 index 0000000000..5d44d4cc93 --- /dev/null +++ b/src/domain/javascript/electronStaticReceiverAdmission.test.ts @@ -0,0 +1,80 @@ +import { expect, it } from "vitest"; + +import { inspectElectronIpcNode } from "./electronStaticAnalysisIpc.js"; +import { resolveElectronBrowserWindowReceiver } from "./electronStaticRenderer.js"; +import { createJavaScriptRpcScan } from "./javascriptRpcEndpoints.js"; +import { + indexJavaScriptBindingsWithReceiversSteps, + indexJavaScriptElectronIdentitySteps, +} from "./javascriptSemanticAnalysis.js"; +import { createSemanticExpressionShapeResolver } from "./javascriptSemanticExpressionShapes.js"; +import { + completeSemanticSteps as complete, + traverseJavaScriptAst, +} from "./javascriptSemanticTraversal.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { + createJavaScriptAnalysisAccumulator, + type JavaScriptFindingContext, +} from "./javascriptStaticAnalysisState.js"; + +const inspect = (source: string) => { + const file = parseJavaScriptSource(source); + if (file === null) throw new Error("Fixture must parse"); + const index = complete( + indexJavaScriptBindingsWithReceiversSteps(file.program), + ); + complete(index.indexAmbientWritesSteps()); + const identity = complete( + indexJavaScriptElectronIdentitySteps(file.program, index.bindings), + ); + let mutationIndexCalls = 0; + const original = index.indexMemberMutationFactsSteps.bind(index); + index.indexMemberMutationFactsSteps = function* () { + mutationIndexCalls += 1; + yield* original(); + }; + const context: JavaScriptFindingContext = { + source, + accumulator: createJavaScriptAnalysisAccumulator(source.length), + vitePreloadDependencyTable: null, + electronBindings: identity.bindings, + electronExportIntact: identity.intactExport, + rpcScan: createJavaScriptRpcScan( + source, + index.receivers.rpcClientRoots, + index, + ), + expressionShapes: createSemanticExpressionShapeResolver(index.bindings), + browserWindowReceiver: (node, site, suffix) => + resolveElectronBrowserWindowReceiver(node, site, suffix, context, index), + }; + traverseJavaScriptAst(file, { + enter: (node) => inspectElectronIpcNode(node, context), + }); + return { findings: context.accumulator.ipc, mutationIndexCalls }; +}; + +it.each([ + [ + "ordinary EventEmitter", + `import {EventEmitter} from "node:events"; + const emitter = new EventEmitter(); emitter.on("event", listener);`, + ], + [ + "unsupported BrowserWindow path", + `const {BrowserWindow} = require("electron"); + const window = new BrowserWindow(); window.on("ready", listener);`, + ], +])("does not request whole-file mutation facts for %s", (_, source) => { + const result = inspect(source); + expect(result.findings).toEqual([]); + expect(result.mutationIndexCalls).toBe(0); +}); + +it("still checks mutations for a source-proven scoped Electron receiver", () => { + const result = inspect(`const {BrowserWindow} = require("electron"); + const window = new BrowserWindow(); window.webContents.ipc.handle("channel", () => {});`); + expect(result.findings).toHaveLength(1); + expect(result.mutationIndexCalls).toBeGreaterThan(0); +}); diff --git a/src/domain/javascript/electronStaticRenderer.ts b/src/domain/javascript/electronStaticRenderer.ts index 86c4128327..578dff96b6 100644 --- a/src/domain/javascript/electronStaticRenderer.ts +++ b/src/domain/javascript/electronStaticRenderer.ts @@ -25,20 +25,51 @@ export const isElectronRendererLoad = ( return false; const method = semanticStaticPropertyKey(callee.property, callee.computed); if (method !== "loadFile" && method !== "loadURL") return false; - let receiver = unwrapJavaScriptExpression(callee.object).node; - const path: string[] = [method]; - if ( - t.isMemberExpression(receiver) && - semanticStaticPropertyKey(receiver.property, receiver.computed) === - "webContents" - ) { - receiver = unwrapJavaScriptExpression(receiver.object).node; - path.unshift("webContents"); - } + const receiver = resolveElectronBrowserWindowReceiver( + callee.object, + site, + [method], + context, + index, + ); + return ( + receiver !== undefined && + (receiver.path.length === 1 || + (receiver.path.length === 2 && receiver.path[0] === "webContents")) + ); +}; + +/** Source allocation identity, not a claim about runtime window/frame identity. */ +export const resolveElectronBrowserWindowReceiver = ( + node: t.Node, + site: t.Node, + suffix: readonly string[], + context: JavaScriptFindingContext, + index: JavaScriptBindingIndex, +): + | { readonly owner: t.NewExpression; readonly path: readonly string[] } + | undefined => { + let receiver = unwrapJavaScriptExpression(node).node; + const path = [...suffix]; const seen = new Set(); - while (t.isIdentifier(receiver)) { + const aliases: { receiver: t.Identifier; path: readonly string[] }[] = []; + for (;;) { + if ( + t.isMemberExpression(receiver) || + t.isOptionalMemberExpression(receiver) + ) { + const member = semanticStaticPropertyKey( + receiver.property, + receiver.computed, + ); + if (member === null) return undefined; + path.unshift(member); + receiver = unwrapJavaScriptExpression(receiver.object).node; + continue; + } + if (!t.isIdentifier(receiver)) break; const binding = index.bindingForReferenceAt(receiver); - if (binding === undefined || seen.has(binding.bindingId)) return false; + if (binding === undefined || seen.has(binding.bindingId)) return undefined; seen.add(binding.bindingId); const initializer = semanticStableInitializer(binding, index.bindings); if ( @@ -47,21 +78,45 @@ export const isElectronRendererLoad = ( !semanticBindingAtRead(binding, receiver, index.bindings.parentsByNode) .hasDefiniteInitializer ) - return false; - completeSemanticSteps(index.indexMemberMutationFactsSteps()); - if (index.referenceMayHaveChangedAt(site, receiver, path)) return false; + return undefined; + aliases.push({ receiver, path: [...path] }); receiver = unwrapJavaScriptExpression(initializer.node).node; } - if (!t.isNewExpression(receiver)) return false; + if (!t.isNewExpression(receiver)) return undefined; const constructor = electronCalleePath(receiver.callee, context); - return ( - constructor?.length === 1 && - constructor[0] === "BrowserWindow" && + if ( + constructor?.length !== 1 || + constructor[0] !== "BrowserWindow" || + !supportedReceiverPath(path) + ) + return undefined; + if ( context.electronExportIntact?.([ "electron", "BrowserWindow", "prototype", ...path, - ]) === true + ]) !== true + ) + return undefined; + // Only source-proven supported receivers justify the whole-file mutation pass. + // Keep every alias's consumed path and check at the original invocation site. + completeSemanticSteps(index.indexMemberMutationFactsSteps()); + for (const alias of aliases) + if (index.referenceMayHaveChangedAt(site, alias.receiver, alias.path)) + return undefined; + return { owner: receiver, path }; +}; + +const supportedReceiverPath = (path: readonly string[]): boolean => { + const operation = path.at(-1); + if (operation === "loadFile" || operation === "loadURL") + return ( + path.length === 1 || (path.length === 2 && path[0] === "webContents") + ); + return ( + path[0] === "webContents" && + ((path.length === 3 && path[1] === "ipc") || + (path.length === 4 && path[1] === "mainFrame" && path[2] === "ipc")) ); }; diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index 530c00605c..2ecf74191e 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -14,7 +14,10 @@ import { traverseJavaScriptAstSteps, } from "./javascriptSemanticTraversal.js"; -import { isElectronRendererLoad } from "./electronStaticRenderer.js"; +import { + isElectronRendererLoad, + resolveElectronBrowserWindowReceiver, +} from "./electronStaticRenderer.js"; import { finishElectronNativeBindingsSteps } from "./electronStaticAnalysisNative.js"; import { inspectElectronStaticNode } from "./electronStaticAnalysis.js"; import { @@ -109,6 +112,8 @@ export function* analyzeParsedJavaScriptStaticSourceSteps( ); const findings: JavaScriptFindingContext = { pathIntrinsics: createStaticPathIntrinsicQuery(index), + browserWindowReceiver: (node, site, suffix) => + resolveElectronBrowserWindowReceiver(node, site, suffix, findings, index), rendererLoad: (callee, site) => isElectronRendererLoad(callee, site, findings, index), source, diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index 505258501a..edbe450d71 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -101,6 +101,13 @@ export interface JavaScriptFindingContext { /** Native path/URL identities established by shared lexical and mutation owners. */ readonly pathIntrinsics?: JavaScriptStaticPathIntrinsicQuery; readonly electronExportIntact?: (path: readonly string[]) => boolean; + readonly browserWindowReceiver?: ( + node: t.Node, + site: t.Node, + suffix: readonly string[], + ) => + | { readonly owner: t.NewExpression; readonly path: readonly string[] } + | undefined; readonly rendererLoad?: (callee: t.Node, site: t.Node) => boolean; readonly rpcScan: JavaScriptRpcScan; /** Shared bounded argv/nullish argument shapes for optional-slot signatures. */ diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts index c4601eadf6..13a528c2f6 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.test.ts @@ -151,3 +151,32 @@ it("preserves partial request targets through the strict worker schema", () => { const serialized = JSON.parse(JSON.stringify(facts)) as unknown; expect(javaScriptWorkerStaticAnalysisSchema.parse(serialized)).toEqual(facts); }); + +it("preserves scoped IPC receiver provenance and rejects malformed scope metadata", () => { + const facts = + analyzeJavaScriptStaticSource(`const {BrowserWindow} = require("electron"); + const window = new BrowserWindow(); window.webContents.mainFrame.ipc.handle("scoped", () => {});`); + expect(facts.electron.ipc).toHaveLength(1); + expect(facts.electron.ipc[0]?.receiver?.kind).toBe("main-frame"); + expect( + javaScriptWorkerStaticAnalysisSchema.parse( + JSON.parse(JSON.stringify(facts)) as unknown, + ), + ).toEqual(facts); + const finding = facts.electron.ipc[0]!; + for (const receiver of [ + { kind: "global", owner_location: finding.receiver?.owner_location }, + { kind: "main-frame" }, + { ...finding.receiver, runtime_frame_id: 42 }, + ]) { + expect( + javaScriptWorkerStaticAnalysisSchema.safeParse({ + ...facts, + electron: { + ...facts.electron, + ipc: [{ ...finding, receiver }], + }, + }).success, + ).toBe(false); + } +}); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index a647147225..88825153c6 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -67,10 +67,17 @@ const locatedHandler = { ]), handler_location: range, }; +const ipcReceiver = z + .strictObject({ + kind: z.enum(["web-contents", "main-frame"]), + owner_location: range, + }) + .exactOptional(); const ipcChannel = (shape: Shape) => z.union([ z.strictObject({ ...findingContext, + receiver: ipcReceiver, invocation_location: range.optional(), resolution_basis: z.literal("wrapper-substitution").optional(), ...shape, @@ -79,6 +86,7 @@ const ipcChannel = (shape: Shape) => }), z.strictObject({ ...findingContext, + receiver: ipcReceiver, invocation_location: range.optional(), resolution_basis: z.literal("wrapper-substitution").optional(), ...shape, diff --git a/tests/boundary/cli/electronScopedIpc.test.ts b/tests/boundary/cli/electronScopedIpc.test.ts new file mode 100644 index 0000000000..78721ce5d4 --- /dev/null +++ b/tests/boundary/cli/electronScopedIpc.test.ts @@ -0,0 +1,114 @@ +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "preserves scoped IPC allocations without global dispatch pairing through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-scoped-ipc-"); + const copy = `const {BrowserWindow}=require("electron"); const window=new BrowserWindow(); window.webContents.ipc.handle("copied",()=>{});`; + await writeFixtureFiles(root, { + "copy-one.cjs": copy, + "copy-two.cjs": copy, + "main.cjs": `const { BrowserWindow, ipcMain } = require("electron"); + const first = new BrowserWindow(); + const second = new BrowserWindow(); + first.webContents.mainFrame.ipc.handle("shared", () => "first"); + second.webContents.mainFrame.ipc.handle("shared", () => "second"); + ipcMain.handle("shared", () => "global"); + const dynamicWindow = new BrowserWindow(); + dynamicWindow.webContents.ipc.handle(dynamicChannel, () => "dynamic"); + ipcMain.handle("dynamic-shadow", () => "global"); + const fake = { webContents: { ipc: { handle() {} } } }; + fake.webContents.ipc.handle("fake", () => {});`, + "preload.cjs": `const { ipcRenderer } = require("electron"); + ipcRenderer.invoke("shared"); + ipcRenderer.invoke("dynamic-shadow");`, + }); + const application = await analyzeJavaScriptDirectory(surface, cli, root); + expect(application.statistics.parse_failures).toBe(0); + expect(application.summary.ipc).toMatchObject({ + operations: 9, + main_handlers: 7, + renderer_transmissions: 2, + paired_renderer_transmissions: 0, + ambiguous_renderer_transmissions: 2, + }); + const channels = application.graph.nodes.filter( + ({ kind }) => kind === "ipc-channel", + ); + const copies = application.graph.nodes.filter( + ({ kind, observations }) => + kind === "ipc-handler" && + observations.some(({ properties }) => properties.channel === "copied"), + ); + expect(copies).toHaveLength(2); + expect( + channels.filter(({ observations }) => + observations.some(({ properties }) => properties.channel === "copied"), + ), + ).toHaveLength(2); + const shared = channels.filter(({ observations }) => + observations.some(({ properties }) => properties.channel === "shared"), + ); + expect(shared).toHaveLength(3); + const scoped = shared.filter(({ observations }) => + observations.some(({ properties }) => properties.receiver !== undefined), + ); + expect(scoped).toHaveLength(2); + const owners = scoped.flatMap(({ observations }) => + observations.map(({ properties }) => properties.receiver), + ); + expect(new Set(owners.map((owner) => JSON.stringify(owner))).size).toBe(2); + expect( + channels.some(({ observations }) => + observations.some(({ properties }) => properties.channel === "fake"), + ), + ).toBe(false); + expect( + application.graph.edges.filter( + ({ evidence }) => + evidence.extractor.operation === "pair-ipc-literal-channel", + ), + ).toEqual([]); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "does not uniquely pair a renderer with exactly one scoped literal handler through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-single-scoped-ipc-"); + await writeFixtureFiles(root, { + "main.cjs": `const { BrowserWindow } = require("electron"); + const window = new BrowserWindow(); + window.webContents.mainFrame.ipc.handle("only-scoped", () => "result");`, + "preload.cjs": `const { ipcRenderer } = require("electron"); + ipcRenderer.invoke("only-scoped");`, + }); + const application = await analyzeJavaScriptDirectory(surface, cli, root); + expect(application.summary.ipc).toMatchObject({ + operations: 2, + main_handlers: 1, + renderer_transmissions: 1, + dynamic_channel_operations: 0, + paired_renderer_transmissions: 0, + ambiguous_renderer_transmissions: 1, + }); + const handlers = application.graph.nodes.filter( + ({ kind }) => kind === "ipc-handler", + ); + expect(handlers).toHaveLength(1); + expect(handlers[0]?.observations[0]?.properties.receiver).toMatchObject({ + kind: "main-frame", + }); + expect( + application.graph.edges.filter( + ({ evidence }) => + evidence.extractor.operation === "pair-ipc-literal-channel", + ), + ).toEqual([]); + }, +); From f196ebf80665107aedc16892af25a1e1f9f26f43 Mon Sep 17 00:00:00 2001 From: Dery Ferdika <114888155+DeryFerd@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:59:10 +0700 Subject: [PATCH 05/37] fix(apple): refuse unpaired surrogates in IPS JSON strings (#6990) The IPS JSON reader decodes \uXXXX escapes with String.fromCharCode and returns raw slices verbatim, so an escaped unpaired surrogate reaches Evidence as-is: malformed text reported under complete coverage, with no refusal and no uncertainty. A crash report's procName or exception message comes from the crashed process, so the escape is reachable input. Refuse the string the way the parser already refuses other malformed content. The check runs on the decoded value, so it covers escaped and unescaped text alike, and a high surrogate followed by its low surrogate escape stays valid text. Validation: RED then green in src/native/apple/losslessJson.test.ts, where the new case previously accepted the escaped surrogate; 311 tests across the apple parser suites pass; tsc --noEmit over the changed set exits 0; oxlint reports 0 errors; oxfmt --check is clean; verify:module-boundaries passes. Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/native/apple/losslessJson.test.ts | 29 +++++++++++++++++++ src/native/apple/losslessJson.ts | 41 ++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/native/apple/losslessJson.test.ts b/src/native/apple/losslessJson.test.ts index 7ad511ee9f..42c4b74588 100644 --- a/src/native/apple/losslessJson.test.ts +++ b/src/native/apple/losslessJson.test.ts @@ -126,3 +126,32 @@ it("refuses nesting beyond the selected depth in kept and skipped values", () => ).toThrow(expect.objectContaining({ reason: "depth" })); expect(parseLosslessJson(deep, 0, undefined, { maxDepth: 20 }).end).toBe(40); }); + +it("refuses JSON string escapes for unpaired surrogates", () => { + for (const text of ['{"a":"\\ud800"}', '{"a":"ok\\udfff"}']) + expect(() => parseLosslessJson(text, 0)).toThrow( + expect.objectContaining({ + message: "JSON string contains an unpaired surrogate", + }), + ); + expect(() => parseLosslessJson('{"a":"\\ud83d\\ude00"}', 0)).not.toThrow(); +}); + +it("applies the surrogate check to skipped strings and nested keys", () => { + const skipped: JsonShape = { kind: "object", members: {} }; + for (const text of [ + '{"extra":"\\ud800"}', + '{"extra":["\\ud800"]}', + '{"extra":{"\\ud800":1}}', + ]) + expect(() => parseLosslessJson(text, 0, skipped)).toThrow( + expect.objectContaining({ + message: "JSON string contains an unpaired surrogate", + }), + ); + expect(parseLosslessJson('{"extra":"ok"}', 0, skipped).value).toEqual({ + kind: "object", + members: {}, + unlisted: [], + }); +}); diff --git a/src/native/apple/losslessJson.ts b/src/native/apple/losslessJson.ts index f3777ad848..451a6a95fc 100644 --- a/src/native/apple/losslessJson.ts +++ b/src/native/apple/losslessJson.ts @@ -254,7 +254,7 @@ class Parser { throw new LosslessJsonFailure( `Expected a JSON object key at offset ${String(this.position)}`, ); - this.skipString(); + this.parseString(); this.skipWhitespace(); this.expect(":"); this.skipValue(depth + 1); @@ -295,7 +295,7 @@ class Parser { } } if (ch === '"') { - this.skipString(); + this.parseString(); return; } if (ch === "-" || (ch !== undefined && isDigit(ch.charCodeAt(0)))) { @@ -309,13 +309,40 @@ class Parser { private parseString(): string { const start = this.position; const escaped = this.skipString(); - if (!escaped) return this.text.slice(start + 1, this.position - 1); - // A decoded JSON string cannot contain more code units than its literal. - // The input's byte allowance bounds this allocation, including dense escapes. - const decoded: unknown = JSON.parse(this.text.slice(start, this.position)); + const decoded: unknown = escaped + ? // A decoded JSON string cannot contain more code units than its literal. + // The input's byte allowance bounds this allocation, including dense escapes. + JSON.parse(this.text.slice(start, this.position)) + : this.text.slice(start + 1, this.position - 1); if (typeof decoded !== "string") throw new LosslessJsonFailure("Expected a JSON string"); - return decoded; + return this.requirePairedSurrogates(decoded); + } + + /** + * JSON admits \uXXXX escapes for unpaired surrogates, but the decoded text is + * not valid Unicode. Evidence carrying one would report malformed text as + * complete, so the parser refuses the string like any other malformed input. + * A high surrogate followed by its low surrogate escape stays valid text. + */ + private requirePairedSurrogates(value: string): string { + for (let index = 0; index < value.length; index += 1) { + const unit = value.charCodeAt(index); + if (unit >= 0xd800 && unit <= 0xdbff) { + const next = index + 1 < value.length ? value.charCodeAt(index + 1) : 0; + if (next < 0xdc00 || next > 0xdfff) + throw new LosslessJsonFailure( + "JSON string contains an unpaired surrogate", + ); + index += 1; + continue; + } + if (unit >= 0xdc00 && unit <= 0xdfff) + throw new LosslessJsonFailure( + "JSON string contains an unpaired surrogate", + ); + } + return value; } private skipString(): boolean { From a891bb80aead9ca22be4f1baf513649b401d6d1f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:08:49 +0800 Subject: [PATCH 06/37] refactor(filesystem): preserve admitted file facts Return the validated descriptor and its original metadata from regular-file admission. Propagate that contract through owned reads, hashes, snapshots, artifact inventory and provider file workflows instead of replacing initial facts with another stat. Keep arbitrary output-handle operations separate. Preserve final descriptor/path identity proofs, borrowed-source verification, BigInt precision, cancellation provenance and failed-close ownership. Remove the obsolete source and owned-ZIP baseline stages and update their consumers and regressions. Tie read completion faults to actual reads instead of the retired duplicate stat; instrument real handles without fake handle wrappers. Validation: fresh runtime/catalog build; focused source, lifecycle, filesystem, CLI/MCP and authored provider-fixture tests; full static checks; docs checks; test discovery; duplication check. Platform-specific cases remain skipped on macOS. No installed provider or device behavior was verified. --- src/adb/AdbInputSnapshot.ts | 14 +- src/android/AndroidTargetSnapshot.ts | 11 +- src/android/JadxConfiguration.ts | 4 +- .../ApktoolWorkspace.lifecycle.test.ts | 10 +- src/apktool/ApktoolWorkspace.ts | 6 +- src/application/BinaryTargetResolver.ts | 32 ++-- src/application/RegularFileRead.ts | 3 +- src/artifacts/ArtifactFileOperation.ts | 11 +- src/artifacts/ArtifactFileSource.ts | 22 +-- src/artifacts/GodotPckArtifactReader.test.ts | 29 +-- .../MachOSliceArtifactReader.test.ts | 4 +- src/artifacts/NativeDmgArtifactReader.ts | 4 +- src/artifacts/RegularFileAdmission.test.ts | 41 ++++ src/artifacts/ZipArtifactReader.test.ts | 6 +- src/artifacts/ZipArtifactReader.ts | 29 ++- src/artifacts/apple/DylibFileAccess.ts | 2 +- .../extraction/ArtifactExtraction.ts | 4 +- src/artifacts/inventory/classify.ts | 11 +- .../inventory/hashStableRootArtifact.ts | 18 +- .../pyinstaller/PyInstallerPyzStore.ts | 18 +- src/cutter/CutterBridgeClient.ts | 3 +- src/electron/ElectronFuseReader.ts | 2 +- src/filesystem/OwnedRegularFile.ts | 9 +- src/filesystem/RegularFile.ts | 6 +- src/filesystem/StableFileHash.ts | 89 ++++----- src/firmware/FirmwareFiles.ts | 100 +++++----- src/flutter/FlutterBuildProvider.ts | 11 +- src/ghidra/GhidraClient.ts | 3 +- src/ghidra/GhidraExecutableHeader.ts | 2 +- src/ghidra/GhidraFileArtifacts.ts | 2 +- src/ghidra/GhidraInstallationProperties.ts | 2 +- src/ghidra/GhidraJavaOptions.ts | 2 +- src/ghidra/GhidraLanguageCatalog.ts | 2 +- src/ghidra/GhidraPspExtension.ts | 100 +++++----- src/ghidra/GhidraTargetSnapshot.ts | 2 +- src/ghidra/GhidraTransport.ts | 2 +- src/gpu/GpuCodeObjectsProvider.ts | 2 +- src/ida/IdaConfiguration.ts | 3 +- src/javascript/recovery/RecoveryFiles.ts | 3 +- src/model/ModelArtifactProvider.ts | 2 +- src/native/AppleDispatchMetadata.ts | 3 +- src/native/LldbReadbackFiles.ts | 2 +- src/native/NativePlistSource.ts | 2 +- src/pcap/PcapSnapshot.ts | 20 +- src/process/ArtifactAnalysisResources.test.ts | 23 ++- src/reference/ReferenceSourceReaderFile.ts | 2 +- src/server/environment/stateIntake.ts | 2 +- .../filesystem/adbInputSnapshot.test.ts | 2 +- .../filesystem/flutterBuildProvider.test.ts | 32 ++-- .../ownedArtifactReaderCleanup.test.ts | 175 ++++++++++-------- .../regularFileCancellation.test.ts | 4 +- .../stableFileHashOwnership.test.ts | 98 ++++++---- .../process/artifactFileOutcomes.test.ts | 28 +-- .../adb/adbHashCleanupConsumers.test.ts | 10 +- .../ghidra/ghidraFileFailure.test.ts | 25 ++- .../kuna/kunaHashOwnerPropagation.test.ts | 68 ++++--- .../kuna/kunaInstallationLifetime.test.ts | 45 +++-- .../binaryTargetAdmissionCancellation.test.ts | 4 + .../regularFileNoFollowFallback.test.ts | 4 +- .../selectedArtifactSourceLifecycle.test.ts | 83 --------- 60 files changed, 658 insertions(+), 600 deletions(-) diff --git a/src/adb/AdbInputSnapshot.ts b/src/adb/AdbInputSnapshot.ts index bc547f2504..b060b91ace 100644 --- a/src/adb/AdbInputSnapshot.ts +++ b/src/adb/AdbInputSnapshot.ts @@ -61,12 +61,13 @@ export const snapshotAdbInput = async ( let primaryCause: unknown; try { - source = new OwnedFileHandle( - await (dependencies.openSource ?? openRegularFile)(sourcePath, { - symlinks: "follow", - signal, - }), - ); + const { handle, initial } = await ( + dependencies.openSource ?? openRegularFile + )(sourcePath, { + symlinks: "follow", + signal, + }); + source = new OwnedFileHandle(handle); const destinationPath = join(root.path, basename(sourcePath)); destination = new OwnedFileHandle( await (dependencies.openDestination ?? open)( @@ -75,7 +76,6 @@ export const snapshotAdbInput = async ( 0o600, ), ); - const initial = await source.handle.stat(); if (!Number.isSafeInteger(initial.size) || initial.size < 0) throw new RangeError( `Regular file size cannot be represented safely: ${sourcePath}`, diff --git a/src/android/AndroidTargetSnapshot.ts b/src/android/AndroidTargetSnapshot.ts index 44e7e3547b..bd0af19aa9 100644 --- a/src/android/AndroidTargetSnapshot.ts +++ b/src/android/AndroidTargetSnapshot.ts @@ -19,6 +19,7 @@ import { RegularFileAdmissionFailure, assertRegularFileActive, sameRegularFileState, + type StableRegularFileDescriptor, } from "../filesystem/RegularFile.js"; import type { AndroidOperation } from "../domain/android/androidAnalysis.js"; @@ -173,9 +174,12 @@ const copyAndroidSnapshot = async ( // Close each descriptor through its owner. A rejected close keeps the // copy outcome and stays retained until that owner is retried; the // partial file is removed only after those close attempts. - const source = adoptSnapshotHandle( - await openSnapshotSource(sourcePath, signal, retain(sourcePath)), + const { handle, initial } = await openSnapshotSource( + sourcePath, + signal, + retain(sourcePath), ); + const source = adoptSnapshotHandle(handle); return await withOwnedFileHandle( sourcePath, source.owner, @@ -184,7 +188,6 @@ const copyAndroidSnapshot = async ( ...retain(sourcePath), }, async () => { - const initial = await source.handle.stat(); assertRegularFileActive(signal); let copiedBytes = 0; const hash = createHash("sha256"); @@ -286,7 +289,7 @@ const openSnapshotSource = async ( retention: { readonly retainFailedClose?: (owner: OwnedFileHandle) => void; }, -): Promise => { +): Promise => { try { return await openRegularFile(path, { symlinks: "follow", signal }); } catch (cause: unknown) { diff --git a/src/android/JadxConfiguration.ts b/src/android/JadxConfiguration.ts index 3ff117927a..ca16c71f6b 100644 --- a/src/android/JadxConfiguration.ts +++ b/src/android/JadxConfiguration.ts @@ -341,7 +341,7 @@ const requireEngineArchive = async ( ), }, options.signal, - async (handle) => { + async ({ handle, initial }) => { try { // Metadata enumeration borrows this descriptor. The file workflow // alone owns admission, final close, cancellation and failed retry. @@ -349,7 +349,7 @@ const requireEngineArchive = async ( jar, "zip", JAR_METADATA_READ_MAX_BYTES, - { handle, initial: await handle.stat() }, + { handle, initial }, ); const missing = new Set([ "com/atxx/jhmcp/JadxSession.class", diff --git a/src/apktool/ApktoolWorkspace.lifecycle.test.ts b/src/apktool/ApktoolWorkspace.lifecycle.test.ts index 69cbffa247..1376c65535 100644 --- a/src/apktool/ApktoolWorkspace.lifecycle.test.ts +++ b/src/apktool/ApktoolWorkspace.lifecycle.test.ts @@ -192,11 +192,17 @@ it.each([false, true])( await fs.writeFile(path, Buffer.from([0xff])); vi.mocked(fs.open).mockImplementation(async (name, flags, mode) => { const handle = await actual.open(name, flags, mode); + const read = handle.read.bind(handle); + let readComplete = false; + handle.read = (async (...args: Parameters) => { + const value = await read(...args); + if (value.bytesRead === 0) readComplete = true; + return value; + }) as typeof handle.read; const stat = handle.stat.bind(handle); - let calls = 0; handle.stat = ((...args: Parameters) => stat(...args).then((value) => { - if (++calls === 3) + if (readComplete) controller.abort({ caller: "after bytes before interpretation" }); return value; })) as typeof handle.stat; diff --git a/src/apktool/ApktoolWorkspace.ts b/src/apktool/ApktoolWorkspace.ts index 111e8c08e8..693876740f 100644 --- a/src/apktool/ApktoolWorkspace.ts +++ b/src/apktool/ApktoolWorkspace.ts @@ -68,9 +68,8 @@ export class ApktoolWorkspace { path, this.#files, signal, - async (source) => { + async ({ handle: source, initial }) => { admitted = true; - const initial = await source.stat(); const output = join(this.path, "input.apk"); const file = await open(output, "wx", 0o600); await withArtifactFileHandle( @@ -180,9 +179,8 @@ export class ApktoolWorkspace { path, this.#files, signal, - async (file) => { + async ({ handle: file, initial }) => { admitted = true; - const initial = await file.stat(); const bytes = Buffer.alloc(budget + 1); let length = 0; while (length < bytes.length) { diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 4f9c1d875f..c7a9dd7c9c 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -26,7 +26,10 @@ import { import { extname } from "node:path"; import { BinaryTargetError } from "../domain/configurationErrors.js"; -import { sameRegularFileState } from "../filesystem/RegularFile.js"; +import { + sameRegularFileState, + type StableRegularFileDescriptor, +} from "../filesystem/RegularFile.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -153,10 +156,10 @@ export const parseBinaryTarget = async ( symlinks: "reject", ...(signal === undefined ? {} : { signal }), }, - async (handle) => { + async (source) => { try { return await readOpenedBinaryTarget( - handle, + source, { path, sourcePath, ...bundle }, options, ); @@ -232,7 +235,7 @@ type TargetResolution = Result< /** Classify and hash the one descriptor admitted by the lifetime owner. */ const readOpenedBinaryTarget = async ( - handle: FileHandle, + source: StableRegularFileDescriptor, selected: { readonly path: string; readonly sourcePath: string; @@ -240,13 +243,12 @@ const readOpenedBinaryTarget = async ( }, options: BinaryTargetResolverOptions, ): Promise => { + const { handle, initial: metadata } = source; const { path, sourcePath, bundleInfoPlist } = selected; const bundle = bundleInfoPlist === undefined ? {} : { bundleInfoPlist }; const hostArchitecture = options.hostArchitecture ?? process.arch; const { targetKind, formatHint, signal, architecture, entryPoint } = options; throwIfTargetResolutionCancelled(signal); - const metadata = await handle.stat(); - throwIfTargetResolutionCancelled(signal); if (formatHint === "raw-binary") { if (targetKind !== undefined && targetKind !== "executable") return err( @@ -366,7 +368,7 @@ const readOpenedBinaryTarget = async ( const artifactFormat = targetKind === "database" ? undefined - : await detectArtifactFormat(path, handle, signal); + : await detectArtifactFormat(path, source, signal); if ( targetKind === "database" || (targetKind === undefined && path.toLowerCase().endsWith(".hop")) @@ -389,7 +391,7 @@ const readOpenedBinaryTarget = async ( : ok({ ...identity, kind: "artifact", format: artifactFormat }); } const detected = await readExecutableMetadata( - handle, + source, hostArchitecture, signal, ); @@ -616,7 +618,7 @@ const throwIfTargetResolutionCancelled = (signal?: AbortSignal): void => { const detectArtifactFormat = async ( path: string, - handle: FileHandle, + source: StableRegularFileDescriptor, signal?: AbortSignal, ): Promise< | Exclude< @@ -633,6 +635,7 @@ const detectArtifactFormat = async ( > | undefined > => { + const { handle, initial } = source; throwIfTargetResolutionCancelled(signal); const lower = path.toLowerCase(); const magic = Buffer.alloc(4096); @@ -664,8 +667,7 @@ const detectArtifactFormat = async ( return "pkg"; if (lower.endsWith(".dmg")) { throwIfTargetResolutionCancelled(signal); - const size = (await handle.stat()).size; - throwIfTargetResolutionCancelled(signal); + const size = initial.size; if (size >= 512) { const trailer = Buffer.alloc(4); const read = await handle.read(trailer, 0, trailer.length, size - 512); @@ -696,7 +698,7 @@ const detectArtifactFormat = async ( if (xmlState === "matched") return "plist"; } if (bytesRead === MAX_XML_PLIST_PROBE_BYTES) { - const { size } = await handle.stat(); + const { size } = initial; throwIfTargetResolutionCancelled(signal); if (size > bytesRead) throw new BinaryTargetError( @@ -837,10 +839,11 @@ const resolveAppBundle = async ( }; const readExecutableMetadata = async ( - handle: FileHandle, + source: StableRegularFileDescriptor, hostArchitecture: NodeJS.Architecture, signal?: AbortSignal, ): Promise> => { + const { handle, initial } = source; throwIfTargetResolutionCancelled(signal); const prefix = Buffer.alloc(4096); const prefixRead = await handle.read(prefix, 0, prefix.length, 0); @@ -848,8 +851,7 @@ const readExecutableMetadata = async ( const bytes = prefix.subarray(0, prefixRead.bytesRead); // Header commitments such as Mach-O load commands and FAT slices are // checked against the whole file, not the probed prefix. - const fileSize = (await handle.stat()).size; - throwIfTargetResolutionCancelled(signal); + const fileSize = initial.size; if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) { let evidence = parseMzHeaderEvidence(bytes, fileSize); let signatureEvidence: diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index fba17dce50..d8e5b82299 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -206,9 +206,8 @@ export const withRegularFileCompletion = async ( ...(signal === undefined ? {} : { signal }), symlinks: options.symlinks ?? "follow", }, - async (handle) => { + async ({ handle, initial: stats }) => { admitted = handle; - const stats = await handle.stat(); assertRegularFileActive(signal); const value = await read(handle, stats); assertRegularFileActive(signal); diff --git a/src/artifacts/ArtifactFileOperation.ts b/src/artifacts/ArtifactFileOperation.ts index 6b88b62d5d..679673dc6e 100644 --- a/src/artifacts/ArtifactFileOperation.ts +++ b/src/artifacts/ArtifactFileOperation.ts @@ -1,5 +1,8 @@ import type { FileHandle } from "node:fs/promises"; -import { RegularFileOperationCancelled } from "../filesystem/RegularFile.js"; +import { + RegularFileOperationCancelled, + type StableRegularFileDescriptor, +} from "../filesystem/RegularFile.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { withOwnedFileHandle, @@ -252,7 +255,7 @@ export const withArtifactRegularFile = async ( ): void; }, signal: AbortSignal | undefined, - operation: (file: FileHandle) => Promise, + operation: (source: StableRegularFileDescriptor) => Promise, symlinks: "follow" | "reject" = "reject", ): Promise => { let cleanupLease: AnalysisCleanupLease | undefined; @@ -272,9 +275,9 @@ export const withArtifactRegularFile = async ( }); }, }, - async (file) => { + async (source) => { assertArtifactReadActive(signal); - return await operation(file); + return await operation(source); }, ); return value; diff --git a/src/artifacts/ArtifactFileSource.ts b/src/artifacts/ArtifactFileSource.ts index 4af676b0a8..2711ad465b 100644 --- a/src/artifacts/ArtifactFileSource.ts +++ b/src/artifacts/ArtifactFileSource.ts @@ -156,28 +156,12 @@ export class ArtifactFileSource { handle: this.admitted.handle, initial: this.admitted.initial, }; - const handle = await openRegularFile(this.path, { + const source = await openRegularFile(this.path, { symlinks: "reject", signal, }); - const owner = new OwnedFileHandle(handle); - this.#owner = owner; - try { - return { handle, initial: await handle.stat() }; - } catch (cause: unknown) { - try { - await owner.close(); - this.#owner = undefined; - } catch (cleanupCause: unknown) { - throw new RegularFileAdmissionFailure( - this.path, - cause, - cleanupCause, - owner, - ); - } - throw cause; - } + this.#owner = new OwnedFileHandle(source.handle); + return source; } catch (cause: unknown) { if (cause instanceof RegularFileAdmissionFailure) this.#owner = cause.owner; diff --git a/src/artifacts/GodotPckArtifactReader.test.ts b/src/artifacts/GodotPckArtifactReader.test.ts index c3e82036e8..7938e82303 100644 --- a/src/artifacts/GodotPckArtifactReader.test.ts +++ b/src/artifacts/GodotPckArtifactReader.test.ts @@ -6,8 +6,8 @@ import { pckU64, type GodotPckMember, } from "../../tests/fixtures/godotPck.js"; -import { rename, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; +import { mkdir, rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { text } from "node:stream/consumers"; import { describe, expect, it } from "vitest"; @@ -241,19 +241,20 @@ describe("GodotPckArtifactReader member streams", () => { }); it("reads members from the admitted descriptor after the path is replaced", async () => { - const path = await writePck(buildGodotPck(3, members)); - const handle = await openRegularFile(path, { symlinks: "reject" }); + const root = await createTestTempDirectory("rea-godot-borrowed-source-"); + const selected = join(root, "selected"); + await mkdir(selected); + const path = join(selected, "game"); + await writeFile(path, buildGodotPck(3, members)); + const descriptor = await openRegularFile(path, { symlinks: "reject" }); + const { handle } = descriptor; try { - const replacement = join(dirname(path), "replacement"); - await writeFile( - replacement, - buildGodotPck(3, [{ path: "other.gd", data: "" }]), - ); - await rename(replacement, path); - const reader = new GodotPckArtifactReader(path, { - handle, - initial: await handle.stat(), - }); + // Replace the selected directory without changing the admitted inode's + // metadata. Moving the file itself would change its validated ctime. + await rename(selected, join(root, "original")); + await mkdir(selected); + await writeFile(path, buildGodotPck(3, [{ path: "other.gd", data: "" }])); + const reader = new GodotPckArtifactReader(path, descriptor); const entries = await list(reader); expect(entries.map(({ path: member }) => member)).toEqual([ "scenes/main.gd", diff --git a/src/artifacts/MachOSliceArtifactReader.test.ts b/src/artifacts/MachOSliceArtifactReader.test.ts index a3630479b3..85c6a2537f 100644 --- a/src/artifacts/MachOSliceArtifactReader.test.ts +++ b/src/artifacts/MachOSliceArtifactReader.test.ts @@ -306,8 +306,8 @@ describe("MachOSliceArtifactReader", () => { { cpu: CPU.arm64, bytes: arm }, ]), ); - const handle = await openRegularFile(binary, { symlinks: "reject" }); - const source = { handle, initial: await handle.stat() }; + const source = await openRegularFile(binary, { symlinks: "reject" }); + const { handle } = source; const runner: NativeCommandRunner = { run: () => Promise.resolve( diff --git a/src/artifacts/NativeDmgArtifactReader.ts b/src/artifacts/NativeDmgArtifactReader.ts index 9a3f9959a1..28a378f5ba 100644 --- a/src/artifacts/NativeDmgArtifactReader.ts +++ b/src/artifacts/NativeDmgArtifactReader.ts @@ -597,10 +597,10 @@ export class NativeDmgArtifactReader implements ArtifactReader { ): Promise { let handle: FileHandle; try { - handle = await openRegularFile(this.path, { + ({ handle } = await openRegularFile(this.path, { symlinks: "follow", ...(signal === undefined ? {} : { signal }), - }); + })); } catch (cause: unknown) { if (cause instanceof RegularFileAdmissionFailure) { this.#verificationProbeHandle = cause.owner; diff --git a/src/artifacts/RegularFileAdmission.test.ts b/src/artifacts/RegularFileAdmission.test.ts index fe1d0fb376..eb1ce843b3 100644 --- a/src/artifacts/RegularFileAdmission.test.ts +++ b/src/artifacts/RegularFileAdmission.test.ts @@ -1,6 +1,8 @@ import type { BigIntStats, Stats } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import { tmpdir } from "node:os"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; import { @@ -10,6 +12,7 @@ import { RegularFileOperationCancelled, sameRegularFileState, } from "../filesystem/RegularFile.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; const { mockOpen } = vi.hoisted(() => ({ mockOpen: vi.fn() })); @@ -20,6 +23,44 @@ vi.mock("node:fs/promises", async (importOriginal) => ({ afterEach(() => mockOpen.mockReset()); +it("returns the original validated metadata snapshot with its actual descriptor", async () => { + const { open } = + await vi.importActual( + "node:fs/promises", + ); + const root = await createTestTempDirectory("rea-admission-snapshot-"); + const path = join(root, "selected.bin"); + const original = Buffer.from("admitted α测试\0\n"); + await writeFile(path, original); + let validated: Stats | undefined; + let opened: FileHandle | undefined; + mockOpen.mockImplementationOnce(async (...args: Parameters) => { + const handle = await open(...args); + opened = handle; + const stat = handle.stat.bind(handle); + vi.spyOn(handle, "stat").mockImplementationOnce(async () => { + validated = await stat(); + return validated; + }); + return handle; + }); + const descriptor = await openRegularFile(path, { symlinks: "reject" }); + try { + expect(descriptor.handle).toBe(opened); + expect(descriptor.initial).toBe(validated); + expect(descriptor.initial.size).toBe(original.length); + await writeFile( + path, + "a later, longer replacement extent on the same inode", + ); + expect(descriptor.initial).toBe(validated); + expect(descriptor.initial.size).toBe(original.length); + expect(descriptor.handle.stat).toHaveBeenCalledOnce(); + } finally { + await descriptor.handle.close(); + } +}); + it.each(["cancellation after open", "non-regular metadata", "stat failure"])( "retains the exact owner when close fails after %s", async (failureCase) => { diff --git a/src/artifacts/ZipArtifactReader.test.ts b/src/artifacts/ZipArtifactReader.test.ts index eb55c6b08f..be60df9835 100644 --- a/src/artifacts/ZipArtifactReader.test.ts +++ b/src/artifacts/ZipArtifactReader.test.ts @@ -62,7 +62,9 @@ const admittedZip = async () => { const writer = new ZipWriter(new Uint8ArrayWriter()); await writer.add("member.txt", new TextReader(content)); await writeFile(path, await writer.close()); - const handle = await openRegularFile(path, { symlinks: "reject" }); + const { handle, initial } = await openRegularFile(path, { + symlinks: "reject", + }); // Keep the actual descriptor, but select the positional overload used by // the ZIP adapter so the spy does not infer FileHandle.read's options form. const positional: { @@ -73,7 +75,7 @@ const admittedZip = async () => { position: number, ): Promise<{ bytesRead: number; buffer: Buffer }>; } = handle; - return { path, content, handle, positional, initial: await handle.stat() }; + return { path, content, handle, positional, initial }; }; describe("ZipArtifactReader positional reads", () => { diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index 3ddd544a6a..dc03966bcf 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -75,12 +75,13 @@ class NodeFileReader extends Reader { } async #initialize(): Promise { + let source: StableRegularFileDescriptor; try { - const handle = - this.admitted?.handle ?? + source = + this.admitted ?? (await openRegularFile(this.path, { symlinks: "reject" })); - this.#handle = handle; - if (this.#ownsHandle) this.#owner = new OwnedFileHandle(handle); + this.#handle = source.handle; + if (this.#ownsHandle) this.#owner = new OwnedFileHandle(source.handle); } catch (cause: unknown) { if (cause instanceof RegularFileAdmissionFailure) { this.#owner = cause.owner; @@ -108,17 +109,15 @@ class NodeFileReader extends Reader { ); throw cause; } - const metadata = await this.#handle.stat(); - if ( - !metadata.isFile() || - (this.admitted !== undefined && - !sameRegularFileState(this.admitted.initial, metadata)) - ) - throw new ArtifactReaderFailure( - "integrity", - `ZIP source identity changed before reading: ${this.path}`, - ); - this.size = metadata.size; + if (this.admitted !== undefined) { + const metadata = await source.handle.stat(); + if (!sameRegularFileState(source.initial, metadata)) + throw new ArtifactReaderFailure( + "integrity", + `ZIP source identity changed before reading: ${this.path}`, + ); + } + this.size = source.initial.size; } override async readUint8Array( diff --git a/src/artifacts/apple/DylibFileAccess.ts b/src/artifacts/apple/DylibFileAccess.ts index d0f0561c9f..8fcff4fe11 100644 --- a/src/artifacts/apple/DylibFileAccess.ts +++ b/src/artifacts/apple/DylibFileAccess.ts @@ -89,7 +89,7 @@ export class DylibFileAccess { path, this.resources, this.signal, - async (file) => { + async ({ handle: file }) => { const initial = await file.stat({ bigint: true }); assertArtifactReadActive(this.signal); const result = await operation(file, initial); diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 914b6eef6c..51304d3380 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -554,8 +554,8 @@ const requireExtractableFormat = async ( path, input.resourceScope, signal, - async (handle) => { - const size = (await handle.stat()).size; + async ({ handle, initial }) => { + const size = initial.size; if (await locateGodotPck(handle, size, signal)) return true; if (format !== "elf" && format !== "pe" && format !== "mach-o") return false; diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 7bcbb9d869..6bc93137b3 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -92,15 +92,15 @@ export const classifyAndHashRootForInventory = async ( ): Promise => { if (directory) return { format: "directory", digest: null, rootSource: undefined }; - const ownedHandle = await openRootArtifact( + const source = await openRootArtifact( path, options.resourceScope, options.signal, ); - const handle = ownedHandle.handle; + const { handle, initial } = source; const owner: ArtifactResourceOwner = { kind: "file-handle", - handle: ownedHandle, + handle: source.owner, resource: `root artifact descriptor for ${path}`, }; let outcome: @@ -110,7 +110,6 @@ export const classifyAndHashRootForInventory = async ( } | { readonly kind: "failed"; readonly cause: unknown }; try { - const initial = await handle.stat(); if (!sameRegularFileState(expectedMetadata, initial)) throw new ArtifactReaderFailure( "integrity", @@ -139,9 +138,7 @@ export const classifyAndHashRootForInventory = async ( value: { format, digest, - rootSource: retainSource - ? { handle, initial, owner: ownedHandle } - : undefined, + rootSource: retainSource ? source : undefined, }, }; } catch (cause: unknown) { diff --git a/src/artifacts/inventory/hashStableRootArtifact.ts b/src/artifacts/inventory/hashStableRootArtifact.ts index 5c5ef95334..29ac8be949 100644 --- a/src/artifacts/inventory/hashStableRootArtifact.ts +++ b/src/artifacts/inventory/hashStableRootArtifact.ts @@ -11,6 +11,7 @@ import { RegularFileAdmissionFailure, openRegularFile, sameRegularFileState, + type StableRegularFileDescriptor, } from "../../filesystem/RegularFile.js"; import { abortIfNeeded, @@ -24,11 +25,12 @@ export const openRootArtifact = async ( path: string, resourceScope: ArtifactResourceScope, signal?: AbortSignal, -): Promise => { +): Promise< + StableRegularFileDescriptor & { readonly owner: OwnedFileHandle } +> => { try { - return new OwnedFileHandle( - await openRegularFile(path, { symlinks: "reject", signal }), - ); + const source = await openRegularFile(path, { symlinks: "reject", signal }); + return { ...source, owner: new OwnedFileHandle(source.handle) }; } catch (cause: unknown) { const primary = cause instanceof RegularFileAdmissionFailure ? cause.cause : cause; @@ -69,13 +71,15 @@ export const hashStableRootArtifact = async ( resourceScope: ArtifactResourceScope, signal?: AbortSignal, ): Promise => { - const owner = await openRootArtifact(path, resourceScope, signal); - const handle = owner.handle; + const { handle, initial, owner } = await openRootArtifact( + path, + resourceScope, + signal, + ); let outcome: | { readonly kind: "completed"; readonly value: HashResult } | { readonly kind: "failed"; readonly cause: unknown }; try { - const initial = await handle.stat(); outcome = { kind: "completed", value: await hashStableRootArtifactHandle(path, handle, initial, signal), diff --git a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts index 30ae081bae..cfc22b2425 100644 --- a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts +++ b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts @@ -9,6 +9,7 @@ import { openRegularFile, RegularFileAdmissionFailure, sameRegularFileState, + type StableRegularFileDescriptor, } from "../../filesystem/RegularFile.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; import { @@ -148,8 +149,8 @@ export class PyInstallerPyzStore { if (decoded?.status === "rejected") throw decoded.reason; await this.release(writer); abortIfNeeded(signal); - const opened = await this.#admit(path, signal); - this.#images.set(opened, { path, initial: await opened.stat() }); + const { handle: opened, initial } = await this.#admit(path, signal); + this.#images.set(opened, { path, initial }); return { handle: opened, start: 0 }; } @@ -176,7 +177,7 @@ export class PyInstallerPyzStore { let verify: (() => Promise) | undefined; let close: ((primary?: unknown) => Promise) | undefined; if (image !== undefined) { - handle = await this.#admit(image.path, signal); + ({ handle } = await this.#admit(image.path, signal)); const opened = handle; verify = () => this.#verifyImage(opened, image); close = async (primary) => { @@ -231,14 +232,17 @@ export class PyInstallerPyzStore { this.#images.clear(); } - async #admit(path: string, signal?: AbortSignal): Promise { + async #admit( + path: string, + signal?: AbortSignal, + ): Promise { try { - const handle = await openRegularFile(path, { + const source = await openRegularFile(path, { symlinks: "reject", signal, }); - this.#retain(path, new OwnedFileHandle(handle)); - return handle; + this.#retain(path, new OwnedFileHandle(source.handle)); + return source; } catch (cause: unknown) { if (cause instanceof RegularFileAdmissionFailure) this.#retain(path, cause.owner); diff --git a/src/cutter/CutterBridgeClient.ts b/src/cutter/CutterBridgeClient.ts index f100bd7418..e3bca91b11 100644 --- a/src/cutter/CutterBridgeClient.ts +++ b/src/cutter/CutterBridgeClient.ts @@ -898,8 +898,7 @@ export class CutterBridgeClient { join(this.#directory, entry), this.#descriptorResources, signal, - async (file) => { - const info = await file.stat(); + async ({ handle: file, initial: info }) => { if ( info.size > MAX_DESCRIPTOR_BYTES || info.uid !== currentUid || diff --git a/src/electron/ElectronFuseReader.ts b/src/electron/ElectronFuseReader.ts index 603152e26a..b0a50cd698 100644 --- a/src/electron/ElectronFuseReader.ts +++ b/src/electron/ElectronFuseReader.ts @@ -151,7 +151,7 @@ export const readElectronFuseImage = async ( path, options?.resources ?? new ArtifactResourceScope(), signal, - async (owned) => { + async ({ handle: owned }) => { const opened = await owned.stat({ bigint: true }); if (!sameRegularFileState(before, opened)) throw new RegularFileChangedError(path); diff --git a/src/filesystem/OwnedRegularFile.ts b/src/filesystem/OwnedRegularFile.ts index f3759bba99..17fb36d8d6 100644 --- a/src/filesystem/OwnedRegularFile.ts +++ b/src/filesystem/OwnedRegularFile.ts @@ -6,6 +6,7 @@ import { RegularFileAdmissionFailure, assertRegularFileActive, RegularFileOperationCancelled, + type StableRegularFileDescriptor, } from "./RegularFile.js"; export type RegularFileOperationOutcome = @@ -145,9 +146,9 @@ export interface OwnedRegularFileOptions extends OwnedFileOperationOptions { export const withOwnedRegularFile = async ( path: string, options: OwnedRegularFileOptions, - operation: (handle: FileHandle) => Promise, + operation: (source: StableRegularFileDescriptor) => Promise, ): Promise => { - const handle = await openRegularFile(path, { + const source = await openRegularFile(path, { symlinks: options.symlinks ?? "follow", signal: options.signal, }).catch((cause: unknown) => { @@ -163,9 +164,9 @@ export const withOwnedRegularFile = async ( }); return withOwnedFileHandle( path, - new OwnedFileHandle(handle), + new OwnedFileHandle(source.handle), options, - operation, + () => operation(source), ); }; diff --git a/src/filesystem/RegularFile.ts b/src/filesystem/RegularFile.ts index 95c675d056..3d318e8318 100644 --- a/src/filesystem/RegularFile.ts +++ b/src/filesystem/RegularFile.ts @@ -90,14 +90,14 @@ export interface StableRegularFileDescriptor { readonly initial: Stats; } -/** Open and verify the selected file descriptor without blocking on pipes. */ +/** Admit a non-blocking regular-file descriptor with its validated initial facts. */ export const openRegularFile = async ( path: string, options: { readonly symlinks: "follow" | "reject"; readonly signal?: AbortSignal | undefined; }, -): Promise => { +): Promise => { assertRegularFileActive(options.signal); const handle = await open( path, @@ -131,7 +131,7 @@ export const openRegularFile = async ( throw new RegularFileChangedError(path); assertRegularFileActive(options.signal); } - return handle; + return { handle, initial: metadata }; } catch (cause: unknown) { try { await owner.close(); diff --git a/src/filesystem/StableFileHash.ts b/src/filesystem/StableFileHash.ts index 26f0f58d19..44d9a01b9b 100644 --- a/src/filesystem/StableFileHash.ts +++ b/src/filesystem/StableFileHash.ts @@ -82,55 +82,58 @@ async function readStableFile( includePrefix: boolean, ): Promise { try { - return await withOwnedRegularFile(path, options, async (handle) => { - const initial = await handle.stat(); - if (!Number.isSafeInteger(initial.size) || initial.size < 0) - throw new RangeError( - `Regular file size cannot be represented safely: ${path}`, + return await withOwnedRegularFile( + path, + options, + async ({ handle, initial }) => { + if (!Number.isSafeInteger(initial.size) || initial.size < 0) + throw new RangeError( + `Regular file size cannot be represented safely: ${path}`, + ); + const hash = createHash("sha256"); + const chunk = Buffer.allocUnsafe(64 * 1024); + const prefix = Buffer.alloc( + includePrefix ? Math.min(STABLE_FILE_PREFIX_BYTES, initial.size) : 0, ); - const hash = createHash("sha256"); - const chunk = Buffer.allocUnsafe(64 * 1024); - const prefix = Buffer.alloc( - includePrefix ? Math.min(STABLE_FILE_PREFIX_BYTES, initial.size) : 0, - ); - let position = 0; - while (position <= initial.size) { + let position = 0; + while (position <= initial.size) { + assertRegularFileActive(options.signal); + const length = Math.min(chunk.length, initial.size - position + 1); + if (length <= 0) break; + const read = await handle.read(chunk, 0, length, position); + if ( + position + read.bytesRead > initial.size || + (read.bytesRead === 0 && position < initial.size) + ) + throw new RegularFileChangedError(path); + assertRegularFileActive(options.signal); + if (read.bytesRead === 0) break; + if (position < prefix.length) { + const prefixBytes = Math.min( + read.bytesRead, + prefix.length - position, + ); + chunk.copy(prefix, position, 0, prefixBytes); + } + hash.update(chunk.subarray(0, read.bytesRead)); + position += read.bytesRead; + } assertRegularFileActive(options.signal); - const length = Math.min(chunk.length, initial.size - position + 1); - if (length <= 0) break; - const read = await handle.read(chunk, 0, length, position); + const [opened, currentPath] = await Promise.all([ + handle.stat(), + stat(path), + ]); if ( - position + read.bytesRead > initial.size || - (read.bytesRead === 0 && position < initial.size) + position !== initial.size || + !sameRegularFileState(initial, opened) || + !sameRegularFileState(initial, currentPath) ) throw new RegularFileChangedError(path); assertRegularFileActive(options.signal); - if (read.bytesRead === 0) break; - if (position < prefix.length) { - const prefixBytes = Math.min( - read.bytesRead, - prefix.length - position, - ); - chunk.copy(prefix, position, 0, prefixBytes); - } - hash.update(chunk.subarray(0, read.bytesRead)); - position += read.bytesRead; - } - assertRegularFileActive(options.signal); - const [opened, currentPath] = await Promise.all([ - handle.stat(), - stat(path), - ]); - if ( - position !== initial.size || - !sameRegularFileState(initial, opened) || - !sameRegularFileState(initial, currentPath) - ) - throw new RegularFileChangedError(path); - assertRegularFileActive(options.signal); - const fingerprint = { sha256: hash.digest("hex"), bytes: position }; - return includePrefix ? { ...fingerprint, prefix } : fingerprint; - }); + const fingerprint = { sha256: hash.digest("hex"), bytes: position }; + return includePrefix ? { ...fingerprint, prefix } : fingerprint; + }, + ); } catch (cause: unknown) { if ( cause instanceof RegularFileCleanupFailure || diff --git a/src/firmware/FirmwareFiles.ts b/src/firmware/FirmwareFiles.ts index 27128e78d9..8bf429a95b 100644 --- a/src/firmware/FirmwareFiles.ts +++ b/src/firmware/FirmwareFiles.ts @@ -93,7 +93,7 @@ export class FirmwareFileAccess { path, this.#resources, signal, - async (handle) => { + async ({ handle }) => { const hash = createHash("sha256"); for await (const chunk of readArtifactFileChunks(handle, { signal })) { assertArtifactReadActive(signal); @@ -125,8 +125,7 @@ export class FirmwareFileAccess { source, this.#resources, signal, - async (handle) => { - const stat = await handle.stat(); + async ({ handle, initial: stat }) => { sourceAdmitted = true; if (stat.size === 0 || stat.size > FIRMWARE_LIMITS.inputBytes) throw firmwareInputFailure(request.operation); @@ -179,7 +178,7 @@ export class FirmwareFileAccess { fullPath, this.#resources, signal, - async (handle) => { + async ({ handle }) => { await pipeline( readArtifactFileChunks(handle, { start: offset, @@ -213,20 +212,24 @@ export class FirmwareFileAccess { signal: AbortSignal, ) { return this.#resources.run(() => - withArtifactRegularFile(entry.path, this.#resources, signal, (handle) => - tree.write( - entry.relativePath, - // SafeOutputTree awaits destination setup before consuming. Defer - // the abort-bound descriptor stream until it has a consumer. - Readable.from( - (async function* () { - assertArtifactReadActive(signal); - yield* readArtifactFileChunks(handle, { signal }); - })(), + withArtifactRegularFile( + entry.path, + this.#resources, + signal, + ({ handle }) => + tree.write( + entry.relativePath, + // SafeOutputTree awaits destination setup before consuming. Defer + // the abort-bound descriptor stream until it has a consumer. + Readable.from( + (async function* () { + assertArtifactReadActive(signal); + yield* readArtifactFileChunks(handle, { signal }); + })(), + ), + { sha256, bytes: entry.size }, + signal, ), - { sha256, bytes: entry.size }, - signal, - ), ), ); } @@ -242,36 +245,41 @@ export class FirmwareFileAccess { signal?: AbortSignal, ): Promise { return this.#resources.run(() => - withArtifactRegularFile(path, this.#resources, signal, async (handle) => { - const bytes = await readBoundedFileBytes( - handle, - FIRMWARE_LIMITS.reportBytes, - ); - if (bytes === undefined) - throw firmwareReportFailure( - operation, - "oversized", - `Report exceeded ${FIRMWARE_LIMITS.reportBytes} bytes`, + withArtifactRegularFile( + path, + this.#resources, + signal, + async ({ handle }) => { + const bytes = await readBoundedFileBytes( + handle, + FIRMWARE_LIMITS.reportBytes, ); - assertArtifactReadActive(signal); - if (bytes.length === 0) - throw firmwareReportFailure(operation, "empty", "Report is empty"); - try { - const value: unknown = JSON.parse( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ), - ); - return value; - } catch (cause: unknown) { - throw firmwareReportFailure( - operation, - "invalid_json", - "Tool report is not valid complete JSON", - cause, - ); - } - }).catch((cause: unknown) => { + if (bytes === undefined) + throw firmwareReportFailure( + operation, + "oversized", + `Report exceeded ${FIRMWARE_LIMITS.reportBytes} bytes`, + ); + assertArtifactReadActive(signal); + if (bytes.length === 0) + throw firmwareReportFailure(operation, "empty", "Report is empty"); + try { + const value: unknown = JSON.parse( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + bytes, + ), + ); + return value; + } catch (cause: unknown) { + throw firmwareReportFailure( + operation, + "invalid_json", + "Tool report is not valid complete JSON", + cause, + ); + } + }, + ).catch((cause: unknown) => { throw mapArtifactFileFailure(cause, (primary) => primary instanceof NonRegularFileReadError ? firmwareReportFailure( diff --git a/src/flutter/FlutterBuildProvider.ts b/src/flutter/FlutterBuildProvider.ts index b4b5ea7fd4..6cecca4028 100644 --- a/src/flutter/FlutterBuildProvider.ts +++ b/src/flutter/FlutterBuildProvider.ts @@ -411,12 +411,11 @@ export class FlutterBuildProvider { // stops the write promptly instead of draining the whole file. let source: OwnedFileHandle; try { - source = this.ownResource( - root, - new OwnedFileHandle( - await openRegularFile(path, { symlinks: "follow", signal }), - ), - ); + const { handle } = await openRegularFile(path, { + symlinks: "follow", + signal, + }); + source = this.ownResource(root, new OwnedFileHandle(handle)); } catch (cause) { if (cause instanceof RegularFileAdmissionFailure) { this.ownResource(root, cause.owner); diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index 19995d0cd7..1b13410dd7 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -616,8 +616,7 @@ export class GhidraClient { path, this.#fileResources, signal, - async (handle) => { - const initial = await handle.stat(); + async ({ handle, initial }) => { assertRegularFileActive(signal); if (!Number.isSafeInteger(initial.size) || initial.size < 0) throw new RangeError("Snapshot size must be a safe integer"); diff --git a/src/ghidra/GhidraExecutableHeader.ts b/src/ghidra/GhidraExecutableHeader.ts index 23a8c2632d..aee59a2259 100644 --- a/src/ghidra/GhidraExecutableHeader.ts +++ b/src/ghidra/GhidraExecutableHeader.ts @@ -30,7 +30,7 @@ export const readGhidraExecutableHeader = async ( ), }, options.signal, - async (file) => { + async ({ handle: file }) => { const observed = await readStableFilePrefix( file, path, diff --git a/src/ghidra/GhidraFileArtifacts.ts b/src/ghidra/GhidraFileArtifacts.ts index fb22034b41..0e68bdbd93 100644 --- a/src/ghidra/GhidraFileArtifacts.ts +++ b/src/ghidra/GhidraFileArtifacts.ts @@ -35,7 +35,7 @@ export const readGhidraProfileArtifact = ( signal: AbortSignal | undefined, interpret: (bytes: Buffer) => Value, ): Promise => - withArtifactRegularFile(path, resources, signal, async (file) => { + withArtifactRegularFile(path, resources, signal, async ({ handle: file }) => { const bytes = await readStableFileBytes(file, path, maximumBytes, { symlinks: "reject", ...(signal === undefined ? {} : { signal }), diff --git a/src/ghidra/GhidraInstallationProperties.ts b/src/ghidra/GhidraInstallationProperties.ts index 3177da1242..830dd7c576 100644 --- a/src/ghidra/GhidraInstallationProperties.ts +++ b/src/ghidra/GhidraInstallationProperties.ts @@ -40,7 +40,7 @@ export const readGhidraInstallationProperties = async ( ), }, options.signal, - async (file) => + async ({ handle: file }) => ( await readStableFileBytes( file, diff --git a/src/ghidra/GhidraJavaOptions.ts b/src/ghidra/GhidraJavaOptions.ts index 251d2a4e1d..4dfd295aa9 100644 --- a/src/ghidra/GhidraJavaOptions.ts +++ b/src/ghidra/GhidraJavaOptions.ts @@ -141,7 +141,7 @@ const readArgumentFile = ( path, resources, signal, - async (file) => { + async ({ handle: file }) => { const data = await readBoundedFileBytes(file, remainingBytes, { signal, cancellationFailure: (reason) => diff --git a/src/ghidra/GhidraLanguageCatalog.ts b/src/ghidra/GhidraLanguageCatalog.ts index 27b6319bd2..fa0d17ea12 100644 --- a/src/ghidra/GhidraLanguageCatalog.ts +++ b/src/ghidra/GhidraLanguageCatalog.ts @@ -82,7 +82,7 @@ const readLanguageDefinitionFile = ( path, resources, signal, - async (file) => { + async ({ handle: file }) => { const bytes = await readBoundedFileBytes(file, MAX_LDEFS_BYTES, { signal, cancellationFailure: (reason) => diff --git a/src/ghidra/GhidraPspExtension.ts b/src/ghidra/GhidraPspExtension.ts index c85aeb9599..c3e9b9feb2 100644 --- a/src/ghidra/GhidraPspExtension.ts +++ b/src/ghidra/GhidraPspExtension.ts @@ -116,58 +116,62 @@ const fingerprint = async ( for (const relative of files) { assertArtifactReadActive(signal); const path = join(root, relative); - await withArtifactRegularFile(path, resources, signal, async (handle) => { - const before = await handle.stat(); - if ( - !Number.isSafeInteger(before.size) || - before.size > MAX_BYTES - bytesRead - ) - throw new Error( - "Allegrex extension bytes exceed the inspection limit.", - ); - if (relative === "extension.properties" && before.size > 65536) - throw new Error( - "Allegrex extension properties exceed the inspection limit.", - ); - hash.update(JSON.stringify([relative, before.size])); - const buffer = Buffer.alloc(Math.min(65536, before.size + 1)); - let offset = 0; - const chunks: Buffer[] = []; - // Serial reads keep final close behind every native read. One extra byte - // detects growth without a stream's read-ahead or unbounded allocation. - while (offset <= before.size) { - assertArtifactReadActive(signal); - const { bytesRead } = await handle.read( - buffer, - 0, - Math.min(buffer.length, before.size - offset + 1), - offset, - ); - assertArtifactReadActive(signal); - if (bytesRead === 0) break; - offset += bytesRead; - if (offset > before.size) + await withArtifactRegularFile( + path, + resources, + signal, + async ({ handle, initial: before }) => { + if ( + !Number.isSafeInteger(before.size) || + before.size > MAX_BYTES - bytesRead + ) + throw new Error( + "Allegrex extension bytes exceed the inspection limit.", + ); + if (relative === "extension.properties" && before.size > 65536) + throw new Error( + "Allegrex extension properties exceed the inspection limit.", + ); + hash.update(JSON.stringify([relative, before.size])); + const buffer = Buffer.alloc(Math.min(65536, before.size + 1)); + let offset = 0; + const chunks: Buffer[] = []; + // Serial reads keep final close behind every native read. One extra byte + // detects growth without a stream's read-ahead or unbounded allocation. + while (offset <= before.size) { + assertArtifactReadActive(signal); + const { bytesRead } = await handle.read( + buffer, + 0, + Math.min(buffer.length, before.size - offset + 1), + offset, + ); + assertArtifactReadActive(signal); + if (bytesRead === 0) break; + offset += bytesRead; + if (offset > before.size) + throw new Error( + `Allegrex component changed during inspection: ${relative}`, + ); + const chunk = buffer.subarray(0, bytesRead); + hash.update(chunk); + if (relative === "extension.properties") + chunks.push(Buffer.from(chunk)); + } + if (offset !== before.size) + throw new Error(`Truncated Allegrex component: ${relative}`); + if ( + !sameRegularFileState(before, await handle.stat()) || + !sameRegularFileState(before, await lstat(path)) + ) throw new Error( `Allegrex component changed during inspection: ${relative}`, ); - const chunk = buffer.subarray(0, bytesRead); - hash.update(chunk); + bytesRead += before.size; if (relative === "extension.properties") - chunks.push(Buffer.from(chunk)); - } - if (offset !== before.size) - throw new Error(`Truncated Allegrex component: ${relative}`); - if ( - !sameRegularFileState(before, await handle.stat()) || - !sameRegularFileState(before, await lstat(path)) - ) - throw new Error( - `Allegrex component changed during inspection: ${relative}`, - ); - bytesRead += before.size; - if (relative === "extension.properties") - properties = Buffer.concat(chunks); - }); + properties = Buffer.concat(chunks); + }, + ); } // Ghidra's extension.properties version is the compatible Ghidra version, // not the extension release (e.g. v21.4); content identity supplies the latter. diff --git a/src/ghidra/GhidraTargetSnapshot.ts b/src/ghidra/GhidraTargetSnapshot.ts index c6846bfc5f..0b630c8726 100644 --- a/src/ghidra/GhidraTargetSnapshot.ts +++ b/src/ghidra/GhidraTargetSnapshot.ts @@ -84,7 +84,7 @@ export const createGhidraTargetSnapshot = async ( sourcePath, resources, options.signal, - async (source) => { + async ({ handle: source }) => { const initial = await source.stat({ bigint: true }); assertArtifactReadActive(options.signal); return await withArtifactFileHandle( diff --git a/src/ghidra/GhidraTransport.ts b/src/ghidra/GhidraTransport.ts index 3fbf752efd..e02cc2f3dc 100644 --- a/src/ghidra/GhidraTransport.ts +++ b/src/ghidra/GhidraTransport.ts @@ -102,7 +102,7 @@ export const observeGhidraEndpoint = async ( endpoint.path, resources, options.signal, - async (file) => { + async ({ handle: file }) => { const initial = await file.stat({ bigint: true }); assertArtifactReadActive(options.signal); if (initial.size > BigInt(MAX_TCP_ENDPOINT_BYTES)) diff --git a/src/gpu/GpuCodeObjectsProvider.ts b/src/gpu/GpuCodeObjectsProvider.ts index 3a23dd507d..78c87e72d1 100644 --- a/src/gpu/GpuCodeObjectsProvider.ts +++ b/src/gpu/GpuCodeObjectsProvider.ts @@ -102,7 +102,7 @@ export class GpuCodeObjectsProvider implements GpuCodeObjectsPort { input.path, this.#resources, options?.signal, - (file) => + ({ handle: file }) => this.#read(file, input, options?.signal, (value) => { progress = value; }), diff --git a/src/ida/IdaConfiguration.ts b/src/ida/IdaConfiguration.ts index 400720998c..a85956c9bb 100644 --- a/src/ida/IdaConfiguration.ts +++ b/src/ida/IdaConfiguration.ts @@ -108,8 +108,7 @@ export const readIdaConfiguration = ( ); }, }, - async (handle) => { - const metadata = await handle.stat(); + async ({ handle, initial: metadata }) => { if (metadata.size > MAXIMUM_REGISTRATION_BYTES) throw registrationTooLarge(); const bytes = Buffer.alloc(MAXIMUM_REGISTRATION_BYTES + 1); diff --git a/src/javascript/recovery/RecoveryFiles.ts b/src/javascript/recovery/RecoveryFiles.ts index d5793dde9c..31cac69a0b 100644 --- a/src/javascript/recovery/RecoveryFiles.ts +++ b/src/javascript/recovery/RecoveryFiles.ts @@ -65,9 +65,8 @@ const scanRecoveryFile = async ( path, resources ?? new ArtifactResourceScope(), signal, - async (handle) => { + async ({ handle, initial: before }) => { const hash = createHash("sha256"); - const before = await handle.stat(); if (before.size > maximum) throw new RecoveryFileSizeError( `Expected a regular file no larger than ${String(maximum)} bytes: ${path}`, diff --git a/src/model/ModelArtifactProvider.ts b/src/model/ModelArtifactProvider.ts index 023219995b..9f021a3bfa 100644 --- a/src/model/ModelArtifactProvider.ts +++ b/src/model/ModelArtifactProvider.ts @@ -103,7 +103,7 @@ export class ModelArtifactProvider implements ModelArtifactPort { input.path, this.#resources, options?.signal, - (file) => + ({ handle: file }) => this.#read(file, input, options?.signal, (progress) => { partialProgress = progress; }), diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index ce249c1ca5..0c871d586b 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -364,8 +364,7 @@ export const inspectAppleDispatchMetadata = async ( target.path, resources, signal, - async (handle) => { - const before = await handle.stat(); + async ({ handle, initial: before }) => { if (before.size > maximumBytes) throw inputBudgetExceeded(admission, before.size); if (signal?.aborted) throw cancelled(); diff --git a/src/native/LldbReadbackFiles.ts b/src/native/LldbReadbackFiles.ts index 9914a893ef..2c91f82a6d 100644 --- a/src/native/LldbReadbackFiles.ts +++ b/src/native/LldbReadbackFiles.ts @@ -61,7 +61,7 @@ export class LldbReadbackFiles { path, this.#resources, undefined, - async (handle) => { + async ({ handle }) => { const value = await read(handle); completed = { value }; return value; diff --git a/src/native/NativePlistSource.ts b/src/native/NativePlistSource.ts index 4fa1d7c859..f7609d9db7 100644 --- a/src/native/NativePlistSource.ts +++ b/src/native/NativePlistSource.ts @@ -283,7 +283,7 @@ const readPlistFile = async ( input.selection.path, input.resources, input.signal, - read, + ({ handle }) => read(handle), "follow", ), ); diff --git a/src/pcap/PcapSnapshot.ts b/src/pcap/PcapSnapshot.ts index 5211d85284..3c62ef7c50 100644 --- a/src/pcap/PcapSnapshot.ts +++ b/src/pcap/PcapSnapshot.ts @@ -127,11 +127,11 @@ export class PcapSnapshot { await mkdir(this.configurationPath, { mode: 0o700 }); const source = await sourceRead(async () => { try { - return this.own( - new OwnedFileHandle( - await openRegularFile(path, { symlinks: "follow", signal }), - ), - ); + const { handle } = await openRegularFile(path, { + symlinks: "follow", + signal, + }); + return this.own(new OwnedFileHandle(handle)); } catch (cause) { if (cause instanceof RegularFileAdmissionFailure) { this.own(cause.owner); @@ -240,11 +240,11 @@ export class PcapSnapshot { signal?: AbortSignal, ): Promise { try { - const file = this.own( - new OwnedFileHandle( - await openRegularFile(this.path, { symlinks: "reject", signal }), - ), - ); + const { handle } = await openRegularFile(this.path, { + symlinks: "reject", + signal, + }); + const file = this.own(new OwnedFileHandle(handle)); const before = await file.handle.stat({ bigint: true }); const buffer = Buffer.alloc(64 * 1024); const hash = createHash("sha256"); diff --git a/src/process/ArtifactAnalysisResources.test.ts b/src/process/ArtifactAnalysisResources.test.ts index 025a8a9267..4a1648577b 100644 --- a/src/process/ArtifactAnalysisResources.test.ts +++ b/src/process/ArtifactAnalysisResources.test.ts @@ -514,14 +514,19 @@ it.each([false, true])( }); if (!failedClose) throw primary; if (scope === undefined) throw new Error("Expected retained file scope"); - return withArtifactRegularFile(path, scope, _signal, async (file) => { - const close = file.close.bind(file); - file.close = async () => { - if (!mayClose) throw new Error("injected descriptor close failure"); - await close(); - }; - throw primary; - }); + return withArtifactRegularFile( + path, + scope, + _signal, + async ({ handle: file }) => { + const close = file.close.bind(file); + file.close = async () => { + if (!mayClose) throw new Error("injected descriptor close failure"); + await close(); + }; + throw primary; + }, + ); }; const resources = new ArtifactAnalysisResources( "fixture", @@ -1017,7 +1022,7 @@ it("retains producer output when a completed read is cancelled during failed clo path, scope, caller.signal, - async (file) => { + async ({ handle: file }) => { const close = file.close.bind(file); file.close = async () => { caller.abort(reason); diff --git a/src/reference/ReferenceSourceReaderFile.ts b/src/reference/ReferenceSourceReaderFile.ts index 14ad08cddf..c4d8def722 100644 --- a/src/reference/ReferenceSourceReaderFile.ts +++ b/src/reference/ReferenceSourceReaderFile.ts @@ -248,7 +248,7 @@ const readStableFileEntry = async ( parentBefore.message, safeSize(expected.size), ); - const handle = await openRegularFile(absolute, { + const { handle } = await openRegularFile(absolute, { symlinks: "reject", signal, }); diff --git a/src/server/environment/stateIntake.ts b/src/server/environment/stateIntake.ts index f5ffeb8d3a..86112345c7 100644 --- a/src/server/environment/stateIntake.ts +++ b/src/server/environment/stateIntake.ts @@ -175,7 +175,7 @@ const readStateFile = async ( handle: owner, }), }, - (handle) => + ({ handle }) => readStableFileBytes(handle, path, STATE_INTAKE_LIMIT_BYTES, { symlinks: "follow", }), diff --git a/tests/boundary/filesystem/adbInputSnapshot.test.ts b/tests/boundary/filesystem/adbInputSnapshot.test.ts index 36bb8e82ae..2308c768a6 100644 --- a/tests/boundary/filesystem/adbInputSnapshot.test.ts +++ b/tests/boundary/filesystem/adbInputSnapshot.test.ts @@ -33,7 +33,7 @@ it("retains and retries a source handle that fails close during admission", asyn undefined, { openSource: async (path, options) => { - const handle = await openRegularFile(path, options); + const { handle } = await openRegularFile(path, options); const close = handle.close.bind(handle); const owner = new OwnedFileHandle(handle); handle.close = async () => { diff --git a/tests/boundary/filesystem/flutterBuildProvider.test.ts b/tests/boundary/filesystem/flutterBuildProvider.test.ts index b3ee5e1706..9d7176f0ca 100644 --- a/tests/boundary/filesystem/flutterBuildProvider.test.ts +++ b/tests/boundary/filesystem/flutterBuildProvider.test.ts @@ -1463,10 +1463,11 @@ it.each(["EACCES", "EPERM"] as const)( const admitted = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (...args) => { - const handle = await original(...args); + const descriptor = await original(...args); + const { handle } = descriptor; if (stage === "read") vi.spyOn(handle, "read").mockRejectedValueOnce(cause); - return handle; + return descriptor; }); const pathStat = stage === "revalidate" @@ -1498,21 +1499,23 @@ it("detects same-size source rewrites with restored mtime through the opened des const admitted = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (...args) => { - const handle = await nativeAdmission(...args); - // Admission returns an open descriptor; mutate its file before the final - // descriptor stat, preserving the size and restoring the visible mtime. - const nativeStat = handle.stat.bind(handle); - let calls = 0; - vi.spyOn(handle, "stat").mockImplementation(async (...statArgs) => { - calls += 1; - if (calls === 2) { + const descriptor = await nativeAdmission(...args); + const { handle } = descriptor; + // Rewrite after a real descriptor read. The retained admission snapshot + // remains the baseline for final verification, including ctime. + const nativeRead = handle.read.bind(handle); + let rewritten = false; + vi.spyOn(handle, "read").mockImplementation(async (...readArgs) => { + const result = await nativeRead(...readArgs); + if (!rewritten) { + rewritten = true; await new Promise((resolve) => setTimeout(resolve, 10)); await writeFile(path, content); await utimes(path, before.atime, before.mtime); } - return nativeStat(...statArgs); + return result; }); - return handle; + return descriptor; }); onTestFinished(() => admitted.mockRestore()); const provider = new FlutterBuildProvider(); @@ -1560,14 +1563,15 @@ it("stops a cancelled descriptor copy before writing its next chunk and releases const admitted = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (...args) => { - const handle = await nativeAdmission(...args); + const descriptor = await nativeAdmission(...args); + const { handle } = descriptor; const nativeRead = handle.read.bind(handle); vi.spyOn(handle, "read").mockImplementation(async (...readArgs) => { const result = await nativeRead(...readArgs); controller.abort(new Error("cancelled while copy read was pending")); return result; }); - return handle; + return descriptor; }); onTestFinished(() => admitted.mockRestore()); const removeRoot = vi.fn((root: string) => diff --git a/tests/boundary/filesystem/ownedArtifactReaderCleanup.test.ts b/tests/boundary/filesystem/ownedArtifactReaderCleanup.test.ts index 8d0896a6cd..f33b1d7e2c 100644 --- a/tests/boundary/filesystem/ownedArtifactReaderCleanup.test.ts +++ b/tests/boundary/filesystem/ownedArtifactReaderCleanup.test.ts @@ -47,95 +47,108 @@ const runner: NativeCommandRunner = { }), }; -const readers = [ - ["ZIP", (path: string) => new ZipArtifactReader(path, "zip")], - [ - "Mach-O slice", - (path: string) => new MachOSliceArtifactReader(path, {}, runner), - ], +const scenarios = [ + { + reader: "ZIP", + createReader: (path: string) => new ZipArtifactReader(path, "zip"), + stage: "admission", + failOnStat: 1, + }, + { + reader: "Mach-O slice", + createReader: (path: string) => + new MachOSliceArtifactReader(path, {}, runner), + stage: "admission", + failOnStat: 1, + }, + { + reader: "Mach-O slice", + createReader: (path: string) => + new MachOSliceArtifactReader(path, {}, runner), + stage: "post-admission-verification", + failOnStat: 2, + }, ] as const; -it.each(readers)( - "retains %s reader admission and stat owners", - async (_, createReader) => { +it.each(scenarios)( + "retains $reader reader $stage owner", + async ({ createReader, stage, failOnStat: initialFailOnStat }) => { const root = await createTestTempDirectory("rea-owned-reader-cleanup-"); const path = join(root, "source"); await writeFile(path, "fixture source"); - for (const stage of ["admission", "post-open-stat"] as const) { - const statFailure = Object.assign( - new Error(`primary ${stage} stat failure`), - { code: "EIO", errno: -5, syscall: "fstat" }, - ); - const closeFailure = new Error( - `close failed after invalidation during ${stage}`, - ); - let descriptor: FileHandle | undefined; - let statCalls = 0; - let closeAttempts = 0; - let failOnStat = stage === "admission" ? 1 : 2; - injected.opened = (openedPath, handle) => { - if (openedPath !== path || descriptor !== undefined) return; - descriptor = handle; - const stat = handle.stat.bind(handle); - const close = handle.close.bind(handle); - vi.spyOn(handle, "stat").mockImplementation(async (...args) => { - statCalls += 1; - if (statCalls === failOnStat) { - failOnStat = -1; - throw statFailure; - } - return stat(...args); - }); - vi.spyOn(handle, "close").mockImplementation(async () => { - closeAttempts += 1; - await close(); - throw closeFailure; - }); - }; - - const reader = createReader(path); - const scope = new ArtifactResourceScope(); - const owner = { kind: "reader" as const, resource: path, reader }; - onTestFinished(async () => { - injected.opened = undefined; - await scope.close().catch(() => undefined); - vi.restoreAllMocks(); - }); - - const primary = await (async () => { - try { - for await (const _entry of reader.entries()) { - // Both injected failures happen before the reader can yield an entry. - } - } catch (cause: unknown) { - return cause; + const statFailure = Object.assign( + new Error(`primary ${stage} stat failure`), + { code: "EIO", errno: -5, syscall: "fstat" }, + ); + const closeFailure = new Error( + `close failed after invalidation during ${stage}`, + ); + let descriptor: FileHandle | undefined; + let statCalls = 0; + let closeAttempts = 0; + let failOnStat: number = initialFailOnStat; + injected.opened = (openedPath, handle) => { + if (openedPath !== path || descriptor !== undefined) return; + descriptor = handle; + const stat = handle.stat.bind(handle); + const close = handle.close.bind(handle); + vi.spyOn(handle, "stat").mockImplementation(async (...args) => { + statCalls += 1; + if (statCalls === failOnStat) { + failOnStat = -1; + throw statFailure; } - throw new Error("Expected reader initialization to fail"); - })(); - expect(primary).toBeInstanceOf(Error); - expect(causeChainContains(primary, statFailure)).toBe(true); + return stat(...args); + }); + vi.spyOn(handle, "close").mockImplementation(async () => { + closeAttempts += 1; + await close(); + throw closeFailure; + }); + }; - const release = await scope.release(owner); - expect(release).toMatchObject({ - kind: "failed", - cause: closeFailure, - owner, - }); - if (release.kind !== "failed") - throw new Error("Expected owned cleanup failure"); - expect(release.cause).toBe(closeFailure); - expect(release.lease.released).toBe(false); - expect(descriptor?.fd).toBe(-1); - await expect(scope.close()).rejects.toMatchObject({ - reason: "unavailable", - cleanup: { reason: closeFailure.message, resources: [path] }, - cleanupLeases: [release.lease], - }); - // Native close invalidated the fd before rejecting. Retrying the same owner - // must remain failed without another syscall or a false successful release. - expect(closeAttempts).toBe(1); - } + const reader = createReader(path); + const scope = new ArtifactResourceScope(); + const owner = { kind: "reader" as const, resource: path, reader }; + onTestFinished(async () => { + injected.opened = undefined; + await scope.close().catch(() => undefined); + vi.restoreAllMocks(); + }); + + const primary = await (async () => { + try { + for await (const _entry of reader.entries()) { + // Admission or its first fresh verification fails before any entry. + } + } catch (cause: unknown) { + return cause; + } + throw new Error("Expected reader initialization to fail"); + })(); + expect(primary).toBeInstanceOf(Error); + expect(causeChainContains(primary, statFailure)).toBe(true); + + const release = await scope.release(owner); + expect(release).toMatchObject({ + kind: "failed", + cause: closeFailure, + owner, + }); + if (release.kind !== "failed") + throw new Error("Expected owned cleanup failure"); + expect(release.cause).toBe(closeFailure); + expect(release.lease.released).toBe(false); + expect(descriptor?.fd).toBe(-1); + await expect(scope.close()).rejects.toMatchObject({ + reason: "unavailable", + cleanup: { reason: closeFailure.message, resources: [path] }, + cleanupLeases: [release.lease], + }); + // Native close invalidated the fd before rejecting. Retrying the same owner + // must remain failed without another syscall or a false successful release. + expect(closeAttempts).toBe(1); }, ); diff --git a/tests/boundary/filesystem/regularFileCancellation.test.ts b/tests/boundary/filesystem/regularFileCancellation.test.ts index 1f4f9dc2ff..4fb159d10b 100644 --- a/tests/boundary/filesystem/regularFileCancellation.test.ts +++ b/tests/boundary/filesystem/regularFileCancellation.test.ts @@ -359,7 +359,7 @@ it.each([false, true])( return withOwnedRegularFile( path, { signal: inner.signal }, - async (nested) => { + async ({ handle: nested }) => { innerHandle = nested; const value = await nested.readFile(); const close = nested.close.bind(nested); @@ -500,7 +500,7 @@ it.each(["completed", "failed"] as const)( signal: controller.signal, retainFailedClose: (owner) => retained.push(owner), }, - async (handle) => { + async ({ handle }) => { const bytes = await handle.readFile(); if (outcome === "failed") throw primary; return bytes; diff --git a/tests/boundary/filesystem/stableFileHashOwnership.test.ts b/tests/boundary/filesystem/stableFileHashOwnership.test.ts index 02cd4efba8..b4bb528fe2 100644 --- a/tests/boundary/filesystem/stableFileHashOwnership.test.ts +++ b/tests/boundary/filesystem/stableFileHashOwnership.test.ts @@ -20,6 +20,38 @@ import { import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; import type { VerifiedLinuxHopperPackage } from "../../../src/application/LinuxHopperPublisher.js"; +it("rejects a real extent change after admission instead of replacing its baseline", async () => { + const root = await createTestTempDirectory("rea-stable-admitted-extent-"); + const path = join(root, "input.bin"); + const original = Buffer.from("admitted extent"); + await writeFile(path, original); + const nativeOpen = regularFile.openRegularFile; + let descriptor: regularFile.StableRegularFileDescriptor | undefined; + const open = vi + .spyOn(regularFile, "openRegularFile") + .mockImplementationOnce(async (...args) => { + descriptor = await nativeOpen(...args); + await writeFile(path, "a different and longer extent after admission"); + return descriptor; + }); + try { + const failure: unknown = await hashStableFile(path, { + retainFailedClose: () => { + throw new Error("Changed input descriptor must close successfully"); + }, + }).catch((cause: unknown) => cause); + expect(failure).toMatchObject({ _tag: "StableFileHashFailure" }); + if (!(failure instanceof Error)) throw new Error("Expected changed input"); + expect(failure.cause).toBeInstanceOf(regularFile.RegularFileChangedError); + expect(descriptor?.initial.size).toBe(original.length); + expect(descriptor?.handle.fd).toBe(-1); + } finally { + open.mockRestore(); + if (descriptor !== undefined && descriptor.handle.fd >= 0) + await descriptor.handle.close(); + } +}); + it("captures a two-byte prefix across short reads and buffer reuse", async () => { const root = await createTestTempDirectory("rea-stable-observation-prefix-"); const path = join(root, "script"); @@ -30,27 +62,27 @@ it("captures a two-byte prefix across short reads and buffer reuse", async () => const open = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (...args) => { - const handle = await originalOpen(...args); - return { - get fd() { - return handle.fd; - }, - stat: (...statArgs: Parameters) => - handle.stat(...statArgs), - read: ( + const descriptor = await originalOpen(...args); + const { handle } = descriptor; + const positional: { + read( buffer: Buffer, offset: number, length: number, position: number, - ) => - handle.read( + ): Promise<{ bytesRead: number; buffer: Buffer }>; + } = handle; + const nativeRead = positional.read.bind(handle); + vi.spyOn(positional, "read").mockImplementation( + (buffer, offset, length, position) => + nativeRead( buffer, offset, position === 0 ? Math.min(length, 1) : length, position, ), - close: () => handle.close(), - } as unknown as Awaited>; + ); + return descriptor; }); try { const observation = await observeStableFile(path, { @@ -121,12 +153,13 @@ it("preserves a read cancellation cause when close also fails", async () => { const open = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (selectedPath, options) => { - const handle = await nativeOpen(selectedPath, options); + const descriptor = await nativeOpen(selectedPath, options); + const { handle } = descriptor; vi.spyOn(handle, "stat").mockImplementationOnce(async () => { controller.abort(primary); throw primary; }); - return handle; + return descriptor; }); const originalClose = OwnedFileHandle.prototype.close; const close = vi @@ -270,15 +303,10 @@ it("distinguishes a thrown undefined primary failure from a clean hash", async ( const open = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (selectedPath, options) => { - const handle = await openFileAtBoundary(selectedPath, options); - const originalStat = handle.stat.bind(handle); - let calls = 0; - vi.spyOn(handle, "stat").mockImplementation(async () => { - calls += 1; - if (calls === 2) throw undefined; - return originalStat(); - }); - return handle; + const descriptor = await openFileAtBoundary(selectedPath, options); + const { handle } = descriptor; + vi.spyOn(handle, "stat").mockRejectedValueOnce(undefined); + return descriptor; }); const closeCause = new Error("injected close failure after undefined"); const close = vi @@ -454,9 +482,10 @@ it("tags a physical hash failure without inferring cancellation from a later sig const open = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (selected, options) => { - const handle = await nativeOpen(selected, options); + const descriptor = await nativeOpen(selected, options); + const { handle } = descriptor; vi.spyOn(handle, "read").mockRejectedValueOnce(primary); - return handle; + return descriptor; }); const nativeClose = OwnedFileHandle.prototype.close; const close = vi @@ -497,12 +526,13 @@ it.each([false, true])( const open = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (selected, options) => { - const handle = await nativeOpen(selected, options); + const descriptor = await nativeOpen(selected, options); + const { handle } = descriptor; vi.spyOn(handle, "read").mockImplementationOnce(async () => { controller.abort("cancelled at EOF"); return { bytesRead: 0, buffer: Buffer.alloc(0) }; }); - return handle; + return descriptor; }); const nativeClose = OwnedFileHandle.prototype.close; const close = vi.spyOn(OwnedFileHandle.prototype, "close"); @@ -548,11 +578,14 @@ it.each( const controller = new AbortController(); const nativeOpen = regularFile.openRegularFile; let retained: OwnedFileHandle | undefined; - let handleSeen: Awaited> | undefined; + let handleSeen: + | regularFile.StableRegularFileDescriptor["handle"] + | undefined; const open = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (selected, options) => { - const handle = await nativeOpen(selected, options); + const descriptor = await nativeOpen(selected, options); + const { handle } = descriptor; handleSeen = handle; const read = handle.read.bind(handle); vi.spyOn(handle, "read").mockImplementationOnce(async (...args) => { @@ -560,7 +593,7 @@ it.each( controller.abort(reason); return result; }); - return handle; + return descriptor; }); const nativeClose = OwnedFileHandle.prototype.close; const close = vi.spyOn(OwnedFileHandle.prototype, "close"); @@ -605,10 +638,11 @@ it.each(["physical", "completed"] as const)( const open = vi .spyOn(regularFile, "openRegularFile") .mockImplementation(async (selected, options) => { - const handle = await nativeOpen(selected, options); + const descriptor = await nativeOpen(selected, options); + const { handle } = descriptor; if (outcome === "physical") vi.spyOn(handle, "read").mockRejectedValueOnce(primary); - return handle; + return descriptor; }); const nativeClose = OwnedFileHandle.prototype.close; let ownerSeen: OwnedFileHandle | undefined; diff --git a/tests/boundary/process/artifactFileOutcomes.test.ts b/tests/boundary/process/artifactFileOutcomes.test.ts index 012a531a33..3fb9808c59 100644 --- a/tests/boundary/process/artifactFileOutcomes.test.ts +++ b/tests/boundary/process/artifactFileOutcomes.test.ts @@ -154,17 +154,23 @@ const createFileOutcomeFixture = async ({ const scope = args[4]; if (scope === undefined) throw new Error("Expected retained artifact scope"); - return withArtifactRegularFile(args[0], scope, args[2], async (file) => { - handle = file; - const close = file.close.bind(file); - file.close = async () => { - // A later abort must not replace the already observed physical failure. - caller.abort(cancellationReason); - if (!allowed) throw new Error("controlled descriptor close failure"); - await close(); - }; - throw physical; - }); + return withArtifactRegularFile( + args[0], + scope, + args[2], + async ({ handle: file }) => { + handle = file; + const close = file.close.bind(file); + file.close = async () => { + // A later abort must not replace the already observed physical failure. + caller.abort(cancellationReason); + if (!allowed) + throw new Error("controlled descriptor close failure"); + await close(); + }; + throw physical; + }, + ); }, async (limits) => ({ ...limits, file_bytes: 1048576 }), ); diff --git a/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts b/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts index 93b385b98b..c1547b38b0 100644 --- a/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts +++ b/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts @@ -183,12 +183,13 @@ for (const closeFails of [false, true]) { let opened: FileHandle | undefined; vi.spyOn(regularFile, "openRegularFile").mockImplementation( async (selected, options) => { - const file = await nativeOpen(selected, options); + const descriptor = await nativeOpen(selected, options); + const { handle: file } = descriptor; if (selected === path) { opened = file; vi.spyOn(file, "read").mockRejectedValueOnce(primary); } - return file; + return descriptor; }, ); const nativeClose = OwnedFileHandle.prototype.close; @@ -246,9 +247,10 @@ for (const closeFails of [false, true]) { let opened: FileHandle | undefined; vi.spyOn(regularFile, "openRegularFile").mockImplementation( async (selected, options) => { - const handle = await nativeOpen(selected, options); + const descriptor = await nativeOpen(selected, options); + const { handle } = descriptor; if (selected === path) opened = handle; - return handle; + return descriptor; }, ); const nativeClose = OwnedFileHandle.prototype.close; diff --git a/tests/boundary/providers/ghidra/ghidraFileFailure.test.ts b/tests/boundary/providers/ghidra/ghidraFileFailure.test.ts index fdfaa35c80..ab64340eef 100644 --- a/tests/boundary/providers/ghidra/ghidraFileFailure.test.ts +++ b/tests/boundary/providers/ghidra/ghidraFileFailure.test.ts @@ -32,16 +32,21 @@ const retainFailedDescriptor = async (path: string, primary: unknown) => { let handle: FileHandle | undefined; const failure: unknown = await resources .run(() => - withArtifactRegularFile(path, resources, undefined, async (file) => { - handle = file; - const close = file.close.bind(file); - file.close = async () => { - if (!allowClose) - throw new Error("controlled descriptor close refusal"); - await close(); - }; - throw new OwnedRegularFileOperationFailure(primary); - }), + withArtifactRegularFile( + path, + resources, + undefined, + async ({ handle: file }) => { + handle = file; + const close = file.close.bind(file); + file.close = async () => { + if (!allowClose) + throw new Error("controlled descriptor close refusal"); + await close(); + }; + throw new OwnedRegularFileOperationFailure(primary); + }, + ), ) .catch((cause: unknown) => cause); if (!(failure instanceof ArtifactFileCleanupFailure)) { diff --git a/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts b/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts index 47d6775676..28e49e25e0 100644 --- a/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts +++ b/tests/boundary/providers/kuna/kunaHashOwnerPropagation.test.ts @@ -42,6 +42,10 @@ const injected = vi.hoisted(() => ({ } | undefined, handles: [] as import("node:fs/promises").FileHandle[], + nativeCloses: new Map< + import("node:fs/promises").FileHandle, + () => Promise + >(), })); afterEach(async () => { @@ -56,8 +60,10 @@ afterEach(async () => { injected.failReadStage = undefined; injected.abortOnReadFailure = undefined; injected.abortOnClose = undefined; - for (const handle of injected.handles.splice(0)) - if (handle.fd >= 0) await handle.close(); + for (const [handle, close] of injected.nativeCloses) + if (handle.fd >= 0) await close(); + injected.nativeCloses.clear(); + injected.handles.length = 0; }); vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { @@ -69,7 +75,12 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { ...original, openRegularFile: vi.fn( async (...args: Parameters) => { - const handle = await original.openRegularFile(...args); + const descriptor = await original.openRegularFile(...args); + const { handle } = descriptor; + // Test teardown retains native release separately from the producer's + // injected close behavior, including when an assertion fails early. + const nativeClose = handle.close.bind(handle); + injected.nativeCloses.set(handle, nativeClose); const isInstallation = String(args[0]) === injected.installationPath; injected.handles.push(handle); const failNextClose = injected.failNextClose; @@ -83,7 +94,7 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { !failReadStage && !abortOnClose ) - return handle; + return descriptor; injected.failNextClose = false; let readStageMatched = false; let failedClose = false; @@ -103,13 +114,17 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { stage === "fingerprint" || stage === "identity", ); - return { - get fd() { - return handle.fd; - }, - stat: (...statArgs: Parameters) => - handle.stat(...statArgs), - read: ( + const positional: { + read( + buffer: Buffer, + offset: number, + length: number, + position: number, + ): Promise<{ bytesRead: number; buffer: Buffer }>; + } = handle; + const nativeRead = positional.read.bind(handle); + vi.spyOn(positional, "read").mockImplementation( + ( buffer: Buffer, offset: number, length: number, @@ -144,22 +159,23 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { failCloseReadStage === readStage ) readStageMatched = true; - return handle.read(buffer, offset, length, position); + return nativeRead(buffer, offset, length, position); }, - close: async () => { - injected.closeCalls += 1; - if (abortCloseMatched) abortOnClose?.abort(); - if ( - (failNextClose && !failedClose) || - (readStageMatched && injected.failReadClose && !failedClose) || - (injected.holdCloseFailures && !injected.releaseCloseFailures) - ) { - failedClose = true; - throw new Error("injected close fault"); - } - await handle.close(); - }, - } as unknown as typeof handle; + ); + vi.spyOn(handle, "close").mockImplementation(async () => { + injected.closeCalls += 1; + if (abortCloseMatched) abortOnClose?.abort(); + if ( + (failNextClose && !failedClose) || + (readStageMatched && injected.failReadClose && !failedClose) || + (injected.holdCloseFailures && !injected.releaseCloseFailures) + ) { + failedClose = true; + throw new Error("injected close fault"); + } + await nativeClose(); + }); + return descriptor; }, ), }; diff --git a/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts b/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts index 00e5031a42..aded1b35b9 100644 --- a/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts +++ b/tests/boundary/providers/kuna/kunaInstallationLifetime.test.ts @@ -32,31 +32,28 @@ vi.mock("../../../../src/filesystem/RegularFile.js", async (importOriginal) => { ...original, openRegularFile: vi.fn( async (...args: Parameters) => { - const handle = await original.openRegularFile(...args); + const descriptor = await original.openRegularFile(...args); + const { handle } = descriptor; const index = gates.opens++; - return { - get fd() { - return handle.fd; - }, - stat: (...statArgs: Parameters) => - handle.stat(...statArgs), - read: async (...readArgs: Parameters) => { - if (index === 1 && gates.holdIdentityRead) { - gates.identityRead.enter(); - await gates.identityRead.hold; - } - return handle.read(...readArgs); - }, - close: async () => { - if (index === 1 && gates.holdHashClose) { - gates.hashClose.enter(); - await gates.hashClose.hold; - } - if (gates.failClose) - throw new Error("controlled final close refusal"); - await handle.close(); - }, - } as unknown as typeof handle; + const read = handle.read.bind(handle); + const close = handle.close.bind(handle); + vi.spyOn(handle, "read").mockImplementation(async (...readArgs) => { + if (index === 1 && gates.holdIdentityRead) { + gates.identityRead.enter(); + await gates.identityRead.hold; + } + return read(...readArgs); + }); + vi.spyOn(handle, "close").mockImplementation(async () => { + if (index === 1 && gates.holdHashClose) { + gates.hashClose.enter(); + await gates.hashClose.hold; + } + if (gates.failClose) + throw new Error("controlled final close refusal"); + await close(); + }); + return descriptor; }, ), }; diff --git a/tests/process-global/binaryTargetAdmissionCancellation.test.ts b/tests/process-global/binaryTargetAdmissionCancellation.test.ts index 6347346f2b..898afbeea2 100644 --- a/tests/process-global/binaryTargetAdmissionCancellation.test.ts +++ b/tests/process-global/binaryTargetAdmissionCancellation.test.ts @@ -105,8 +105,12 @@ it.each([ "classifies $name cancellation at descriptor stat $checkpoint and closes the file", async ({ checkpoint, reason }) => { const { path, controller } = await fixture(checkpoint, reason); + // A valid selected load reaches the second stat during hash finalization. const resolved = await parseBinaryTarget(path, { signal: controller.signal, + formatHint: "raw-binary", + architecture: "x86", + entryPoint: "0x1000", }); expect(resolved.ok).toBe(false); if (resolved.ok) throw new Error("cancelled target was admitted"); diff --git a/tests/process-global/regularFileNoFollowFallback.test.ts b/tests/process-global/regularFileNoFollowFallback.test.ts index 44ce66c7c0..0e6e549bbd 100644 --- a/tests/process-global/regularFileNoFollowFallback.test.ts +++ b/tests/process-global/regularFileNoFollowFallback.test.ts @@ -46,7 +46,9 @@ it("rejects a symlink when O_NOFOLLOW is unavailable and still follows when requ await linkedReader.close(); } - const followed = await openRegularFile(link, { symlinks: "follow" }); + const { handle: followed } = await openRegularFile(link, { + symlinks: "follow", + }); try { await expect(readFile(followed)).resolves.toEqual(await readFile(target)); } finally { diff --git a/tests/process-global/selectedArtifactSourceLifecycle.test.ts b/tests/process-global/selectedArtifactSourceLifecycle.test.ts index cc76e8ef82..c6d2536e80 100644 --- a/tests/process-global/selectedArtifactSourceLifecycle.test.ts +++ b/tests/process-global/selectedArtifactSourceLifecycle.test.ts @@ -405,86 +405,3 @@ it.each([false, true])( } }, ); - -it.each([false, true])( - "closes an owned Mach-O source before rejecting its initial stat failure (invalidatingCloseFailure=%s)", - async (invalidatingCloseFailure) => { - const fixture = await machoCarrier(); - const statFailure = Object.assign( - new Error("post-admission initial stat failed"), - { - code: "EIO", - errno: -5, - syscall: "fstat", - }, - ); - const cleanupFailure = new Error( - "close rejected after descriptor invalidation", - ); - const originalAdmission = regularFile.openRegularFile; - vi.spyOn(regularFile, "openRegularFile").mockImplementation( - async (...args) => { - const handle = await originalAdmission(...args); - if (args[0] === fixture.path) { - // Admission has completed; the owner's next stat establishes its initial facts. - vi.spyOn(handle, "stat").mockRejectedValueOnce(statFailure); - const close = handle.close.bind(handle); - vi.spyOn(handle, "close").mockImplementation(async () => { - await close(); - if (invalidatingCloseFailure) throw cleanupFailure; - }); - } - return handle; - }, - ); - const run = vi.spyOn(fixture.runner, "run"); - const reader = new MachOSliceArtifactReader( - fixture.path, - {}, - fixture.runner, - ); - try { - const failure = await entries(reader).catch((cause: unknown) => { - const handle = openedHandle(); - expect(handle.close).toHaveBeenCalledOnce(); - expect(handle.fd).toBe(-1); - return cause; - }); - if (!(failure instanceof ArtifactReaderFailure)) - throw new Error("Expected the primary initial-stat failure"); - expect(failure.reason).toBe("io"); - const primary = invalidatingCloseFailure ? failure.cause : failure; - if (!(primary instanceof ArtifactReaderFailure)) - throw new Error("Missing initial-stat primary failure"); - expect(primary.cause).toBe(statFailure); - const handle = openedHandle(); - if (invalidatingCloseFailure) { - expect(failure.cleanup).toEqual({ - reason: cleanupFailure.message, - resources: [fixture.path], - }); - expect( - await Promise.allSettled([reader.close(), reader.close()]), - ).toEqual([ - { status: "rejected", reason: cleanupFailure }, - { status: "rejected", reason: cleanupFailure }, - ]); - await expect(reader.close()).rejects.toBe(cleanupFailure); - } else { - expect(failure.cleanup).toBeUndefined(); - await Promise.all([reader.close(), reader.close()]); - await reader.close(); - } - expect(handle.close).toHaveBeenCalledOnce(); - expect(handle.fd).toBe(-1); - expect(run).not.toHaveBeenCalled(); - expect(control.handles.size).toBe(1); - } finally { - if (invalidatingCloseFailure) - await reader.close().catch((cause: unknown) => { - expect(cause).toBe(cleanupFailure); - }); - else await reader.close(); - } - }, -); From 162bd56c9ee906837106e3de08c8c426af601ffb Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:12:51 +0800 Subject: [PATCH 07/37] refactor(mcp): share pcap summary delivery handling Use the existing named-contract retained-summary handler for pcap capture inspection. Remove its duplicate validation, detail selection, preflight and Evidence projection policy while preserving client inspection separately. Validation: named pcap MCP schema/delivery, parent issue attribution and workspace admission tests; fresh runtime/catalog build and full static/docs checks. Native packet dissection and Linux-only retained-view tests were not verified on this macOS host. --- src/server/registerPcapTools.ts | 37 ++++++--------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/src/server/registerPcapTools.ts b/src/server/registerPcapTools.ts index 90e50f64b3..bed1aaeed0 100644 --- a/src/server/registerPcapTools.ts +++ b/src/server/registerPcapTools.ts @@ -6,16 +6,13 @@ import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import type { Logger } from "pino"; import { createContractToolHandler, - createEvidenceToolHandler, + createRetainedSummaryToolHandler, } from "./contractToolHandler.js"; import { toolRegistrationOptions, applicationToolRegistrationOptions, } from "./toolRegistrationOptions.js"; import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; -import { retainedSummaryDelivery } from "./retainedSummaryDelivery.js"; -import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; -import { err } from "../domain/result.js"; /** Bind pcap handlers to their exact named contracts through admission. */ export const registerPcapTools = ( @@ -40,39 +37,17 @@ export const registerPcapTools = ( handler(clientContract), ); const contract = toolContract("inspect_pcap_capture"); - const delivery = retainedSummaryDelivery(new Set([contract.name]), { - recordEvidence, - evidenceById, - }); - const capture = createEvidenceToolHandler( + const capture = createRetainedSummaryToolHandler( server, - async (rawInput, options) => { - const parsed = contract.inputSchema.safeParse(rawInput); - if (!parsed.success) - return err( - analysisInputErrorFromIssues( - contract.name, - parsed.error.issues, - rawInput, - { cause: parsed.error }, - ), - ); - const { detail, ...input } = parsed.data; - const parameters = { detail }; - const ready = delivery.preflight(contract.name, parameters); - if (!ready.ok) return ready; - const result = await service.execute(contract.name, input, options); - return result.ok - ? delivery.projectEvidence(contract.name, parameters, result.value) - : result; - }, + contract, + (input, options) => service.execute(contract.name, input, options), logger, - recordEvidence, + { recordEvidence, evidenceById }, withAdmittedAnalysis, ); server.registerTool( contract.name, applicationToolRegistrationOptions(contract), - capture(contract), + capture, ); }; From 22b88300d0ec2930060789e97fbd1371a20e9f5e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:12:53 +0800 Subject: [PATCH 08/37] refactor(environment): reuse admitted capture facts in builds Separate typed manifest/capture composition from unknown-input loading. Builds retain the admitted replay facts locally for credential warnings and record counts instead of parsing the same capture again. Keep full captures out of the compact replay source and preserve manifest-first authentication, parent identity checks, failure precedence and cancellation points. Validation: environment source/build/report/scenario/input-snapshot/state workflows and public build CLI; fresh runtime/catalog build, full static/docs, test discovery and duplication checks. No external clients or targets ran. --- .../environment/environmentBuild.ts | 22 +++++++---- .../environment/environmentSource.ts | 37 ++++++++++++++----- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/application/environment/environmentBuild.ts b/src/application/environment/environmentBuild.ts index 8390c7f8ba..f8325eb804 100644 --- a/src/application/environment/environmentBuild.ts +++ b/src/application/environment/environmentBuild.ts @@ -35,9 +35,10 @@ import { type WrittenBundleFile, } from "./environmentExport.js"; import { - loadEnvironmentSource, + composeEnvironmentSource, parseCaptureEvidence, parseEvidenceFile, + parseViewEvidence, } from "./environmentSource.js"; import { prepareServedState } from "./environmentState.js"; @@ -288,9 +289,21 @@ export const buildEnvironment = async ( ENVIRONMENT_BUILD_OPERATION, ); if (!manifest.ok) return manifest; - const source = loadEnvironmentSource( + const parsedManifest = parseViewEvidence( + ENVIRONMENT_BUILD_OPERATION, + "manifest", manifest.value.value, + "environment-manifest", + ); + if (!parsedManifest.ok) return parsedManifest; + const recorded = parseCaptureEvidence( + ENVIRONMENT_BUILD_OPERATION, input.capture.value, + ); + if (!recorded.ok) return recorded; + const source = composeEnvironmentSource( + parsedManifest.value, + recorded.value, ENVIRONMENT_BUILD_OPERATION, ); if (!source.ok) return source; @@ -306,11 +319,6 @@ export const buildEnvironment = async ( name, operation: ENVIRONMENT_EXPORT_OPERATION, }); - const recorded = parseCaptureEvidence( - ENVIRONMENT_BUILD_OPERATION, - input.capture.value, - ); - if (!recorded.ok) return recorded; warnings.push(...credentialScanWarnings(recorded.value)); const packageSource: EnvironmentPackageSource = diff --git a/src/application/environment/environmentSource.ts b/src/application/environment/environmentSource.ts index 95b832ac65..d80831b840 100644 --- a/src/application/environment/environmentSource.ts +++ b/src/application/environment/environmentSource.ts @@ -268,6 +268,11 @@ interface CaptureFacts { readonly scenarioEvents: readonly BrowserScenarioEvent[] | null; } +/** Authenticated capture identity and the facts derived from its validated payload. */ +export interface ParsedEnvironmentCapture extends CaptureFacts { + readonly evidenceId: string; +} + /** Issue paths of a capture payload refusal start here. */ const CAPTURE_PAYLOAD_PATH = ["evidence", "normalized_result"] as const; @@ -365,7 +370,7 @@ export const parseCaptureEvidence = ( operation: string, value: unknown, purpose: "manifest" | "replay" = "replay", -): Result => { +): Result => { const evidence = parseEvidenceFile(operation, "evidence", value); if (!evidence.ok) return evidence; const normalized = immutableJsonSnapshotMember( @@ -445,8 +450,20 @@ export const loadEnvironmentSource = ( if (!manifest.ok) return manifest; const capture = parseCaptureEvidence(operation, evidenceValue); if (!capture.ok) return capture; - const parentId = manifest.value.view.parent_evidence_id; - if (parentId !== capture.value.evidenceId) + return composeEnvironmentSource(manifest.value, capture.value, operation); +}; + +/** Cross-check admitted manifest and capture facts, retaining only the replay source. */ +export const composeEnvironmentSource = ( + manifest: { + readonly view: EnvironmentManifest; + readonly evidenceId: string; + }, + capture: ParsedEnvironmentCapture, + operation: string = ENVIRONMENT_SERVE_OPERATION, +): Result => { + const parentId = manifest.view.parent_evidence_id; + if (parentId !== capture.evidenceId) return err( inputError( operation, @@ -455,18 +472,18 @@ export const loadEnvironmentSource = ( path: ["evidence", "evidence_id"], reason: "invalid_value", expected: parentId, - message: `The manifest cites parent Evidence ${parentId}; the evidence file is ${capture.value.evidenceId}.`, + message: `The manifest cites parent Evidence ${parentId}; the evidence file is ${capture.evidenceId}.`, }, ], "The manifest was projected from a different capture than the supplied evidence file.", ), ); return ok({ - manifest: manifest.value.view, - manifestEvidenceId: manifest.value.evidenceId, - evidenceId: capture.value.evidenceId, - captureKind: capture.value.captureKind, - captureSha256: capture.value.captureSha256, - exchanges: indexExchanges(manifest.value.view, capture.value.records), + manifest: manifest.view, + manifestEvidenceId: manifest.evidenceId, + evidenceId: capture.evidenceId, + captureKind: capture.captureKind, + captureSha256: capture.captureSha256, + exchanges: indexExchanges(manifest.view, capture.records), }); }; From 149b43ee6158862c94044fd24432dca790d54665 Mon Sep 17 00:00:00 2001 From: JANG BYUNGKUN Date: Sun, 25 Oct 2026 05:21:18 +0900 Subject: [PATCH 09/37] fix(il2cpp): link arm64 builds whose registrations were inlined (#6989) * fix(il2cpp): accept arm64 module arrays loaded through adrp/add pairs A Unity 2022.3 macOS Master (LTO) arm64 build keeps no static CodeRegistration, so the module-array fallback that only decoded x86-64 RIP-relative references reported code-registration-not-found. The registration owner now also decodes aligned adrp + add pairs in executable ranges and keeps the same exact-module-set and single-reference rules. A minimal Mach-O fixture failed on the previous reader (47 passed / 1 failed) and passes here with four fail-closed variants (48/48 binary tests). * fix(il2cpp): name the matched instruction form in inferred module-array limits The inferred-array limitation always described RIP-relative LEA/MOV, which is wrong for arm64 results. The provider now names the adrp/add pair on arm64, and the tool contract and docs cover both forms and state that a located array alone does not finish linking a Master build. The new arm64 provider case failed before the wording change (1 passed / 1 failed) and both cases pass now. * fix(il2cpp): accept type tables loaded without a static MetadataRegistration A Unity 2022.3.62f3 macOS Master build inlines MetadataRegistration as well as CodeRegistration, so linking stopped at metadata-registration-not-found after the module array was found. Registration now infers the only field linking reads, the Il2CppType table, from the one run of type-shaped pointers that covers every referenced type index and whose start is loaded by the existing RIP-relative or adrp/add forms; types_field is null and the Evidence is inferred. The new arm64 and x86-64 fixtures failed on the previous reader (52 passed / 2 failed). Four run-selection variants (interior reference, run extended before the loaded address, two loaded runs, run shorter than the referenced indices) keep metadata-registration-not-found; instruction decoding and record-shape checks are not covered by these fixtures. * test(il2cpp): pin adrp page deltas, register pairing and type-record shape Every adrp fixture targeted its own 4 KiB page, so dropping the page delta or its sign extension, the add register check, the type-kind bound or the type-definition bound left all binary tests passing. Fixtures now reach a later and an earlier page, pair an add with a different destination or source register, break the type run with an out-of-range kind or type-definition index, and bound type-table candidates when the metadata references few type indices. Each of those five source mutations now fails 1 to 3 of 85 binary and provider tests. * refactor(il2cpp): share Il2CppType kind decoding and document type-table bounds The type-table fallback decoded the Il2CppType kind and class/value-type kinds separately from the type namer; both now use the Types owner. Zero and unaligned words are rejected before mapping lookup (Master sample metadata-registration search 92 to 68 ms). Docs and the limitation state that types_count is an upper bound, that a run spanning mappings or a small referenced type range can fail, and credit Cpp2IL StrippedCodeRegSupport and the data-side MetadataRegistration searches of Il2CppDumper and Cpp2IL. Master sample output is unchanged (11,031 code addresses, 37,025 type rows). * fix(il2cpp): refuse generic instances when the type table was inferred An inferred type table has no static MetadataRegistration, so the genericMethodTable it would anchor cannot be located. Report generic-methods-unsupported instead of reading from a null field. * fix(il2cpp): fail closed when an inferred type table is not unique Adversarial review found that the type-table fallback could pick another type-shaped pointer table when the real one was unloaded or excluded, could replace a static MetadataRegistration whose counts matched, and accepted a run cut short by a mapping end. Require exactly one long-enough run in the image, ending inside its mapping and loaded by code; skip the fallback when a record's two counts matched; accept only assigned Il2CppTypeEnum kinds; and look records up through the indexed file mapping so many mappings cannot multiply the scan. Docs and limitations now describe the run length as at least every referenced type index rather than an upper bound. * fix(il2cpp): keep the type-table fallback cheap and its refusals exact Review follow-ups: check that an address can lie in a mapping before the throwing indexed lookup, so words pointing into gaps or zero-fill no longer build an exception each (64 MiB: 74 s to under 2 s); treat a run starting at its mapping start like one reaching its end; stop at the second long-enough run instead of a 64-candidate capacity error; ignore ADRP/ADD pairs on register 31; and say whether a counted MetadataRegistration was unusable or no code-loaded run was found. Docs no longer generalize from one build or state tool history they do not source. * fix(il2cpp): judge type-table slots by their loaded value The run scan read stored words, so a dyld-bound slot's placeholder could decide whether a run qualified. Confirm each slot through readSlot and give a run bounded by a bound or unresolved slot an unknown extent, so the placeholder no longer changes the choice. * docs(il2cpp): keep the failure-reason table layout Shorten the two changed meanings so the table keeps its column widths and the diff shows only those rows. * docs(il2cpp): keep the reason table at its formatted width * fix(il2cpp): read type-table record fields through the loader too The previous change confirmed table slots through readSlot but still read each record's kind and definition index from stored bytes. Read them with readUInt32 and readSlot, and treat a record field the loader binds or leaves unresolved like an unresolved slot: the run it bounds has an unknown extent. * perf(il2cpp): share image mappings across loader-aware reads --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/il2cpp-metadata.md | 33 +- .../il2cpp/Il2cppMetadataService.test.ts | 14 +- .../il2cpp/Il2cppMetadataService.ts | 3 +- src/contracts/il2cpp/il2cppToolContracts.ts | 2 +- .../analysisView/il2cppMetadataView.test.ts | 18 + src/domain/analysisView/il2cppMetadataView.ts | 5 + src/domain/il2cpp/il2cppMetadata.ts | 8 +- src/il2cpp/Il2cppBinaryFile.ts | 29 +- src/il2cpp/Il2cppBinaryGenerics.ts | 8 +- src/il2cpp/Il2cppBinaryImage.ts | 53 +-- src/il2cpp/Il2cppBinaryRegistration.ts | 258 ++++++++++- src/il2cpp/Il2cppBinaryTypes.ts | 14 +- src/il2cpp/Il2cppMetadataProvider.ts | 14 +- .../boundary/il2cpp/il2cppBinaryFile.test.ts | 416 ++++++++++++++++++ .../il2cpp/il2cppMappingOwnership.test.ts | 40 +- .../il2cpp/il2cppMetadataProvider.test.ts | 93 ++++ tests/fixtures/il2cpp/binary.ts | 66 ++- 17 files changed, 1006 insertions(+), 68 deletions(-) diff --git a/docs/il2cpp-metadata.md b/docs/il2cpp-metadata.md index d4326aff0f..0b89ddf7a0 100644 --- a/docs/il2cpp-metadata.md +++ b/docs/il2cpp-metadata.md @@ -241,7 +241,7 @@ record addresses remain distinct candidates, and conflicting file mappings remain malformed. Module records establish names through their declared NUL-terminated string pointers. NUL/0xFF padding, spaces, Unicode and suffix-shared storage are supported. A pointer to a longer string does not match just its suffix; a pointer directly into a shared suffix may name it. Exact reversed UTF-8 matching avoids rescanning the file for every name and retains duplicate-name owners until module records resolve them. A contributor investigated a Linux -`GameAssembly.so` from Unity 2023.2.13f1 whose static registration was not located. When no static pair is resolved on x86-64, the reader searches executable ranges for bytes consistent with RIP-relative LEA/MOV references to module-array candidates. Exactly one referenced array candidate must remain. `code_gen_modules_field` is then `null`, and Evidence is **inferred**: instruction boundaries, the absence of a static registration elsewhere, and runtime use were not verified. The module records and slots themselves remain file observations. Static-table associations are **derived**. Otherwise the result is `code-registration-not-found`. +`GameAssembly.so` from Unity 2023.2.13f1 whose static registration was not located. When no static pair is resolved on x86-64 or arm64, the reader searches executable ranges for bytes consistent with a reference to a module-array candidate: RIP-relative LEA/MOV on x86-64, an adjacent `adrp`/`add` pair on arm64. Exactly one referenced array candidate must remain. `code_gen_modules_field` is then `null`, and Evidence is **inferred**: instruction boundaries, the absence of a static registration elsewhere, and runtime use were not verified. The module records and slots themselves remain file observations. Static-table associations are **derived**. Otherwise the result is `code-registration-not-found`. Cpp2IL's StrippedCodeRegSupport plugin also recovers the module list of an inlined CodeRegistration from the module names; this fallback additionally requires the exact located module set and a single code reference. A Unity 2022.3.62f3 macOS Master build checked by a contributor keeps neither registration symbol; its type table is then inferred as described below. The binary/metadata pair is caller-selected. Name/count/token agreement is structural evidence, not a cryptographic same-build proof. File-offset locations refer to the primary metadata artifact; linked virtual addresses refer to `binary.artifact`. Both SHA-256 identities remain available in compact native pages as `artifact` and `binary_source`. @@ -299,6 +299,30 @@ The result adds: `Il2CppMetadataRegistration.typesCount` and its value. That record is found as the single place whose `fieldOffsetsCount` and `typeDefinitionsSizesCount` both equal the metadata type-definition count and whose type table is file mapped. +Linking reads only that table. When no record on x86-64 or arm64 has both of +those counts, the reader collects maximal runs of pointers to Il2CppType-shaped +records (an assigned Il2CppTypeEnum kind from 0x01 to 0x1e; class and +value-type data below the type-definition count) that are long enough for every +type index the metadata references. Exactly one such run may exist in the +image, it must start and end inside its mapping (a run touching either end may +continue in a neighbouring mapping), and its start must be loaded by the same +instruction forms as the module-array fallback. `types_field` is then `null`, +`types_count` is that run's length rather than an observed `typesCount`, and +Evidence is **inferred**. The run length is at least every referenced type +index but can differ from `typesCount`: a type-shaped pointer after the table +would lengthen it, and an unreadable slot (such as an unresolved relocation) +would end it early. Only the type indices the metadata references are read. A +second long-enough run anywhere, a run that a type-shaped neighbour extends +before the loaded address, a run touching a mapping end or a slot the loader +binds or leaves unresolved (its stored word is only a placeholder), and a covered +128-byte record whose two counts match but whose other fields are unusable all +leave `metadata-registration-not-found` rather than a guess; the search stops +at the second run. The length requirement only separates the type table from +other pointer runs when the metadata references most of it: on the one Master +build checked, the highest referenced index is 6,031 of 6,576 entries. A +metadata referencing few type indices usually +leaves several short runs and fails closed. + Counts use their declared 32-bit width independently of alignment padding. The complete 128-byte registration must have unambiguous loader coverage. Its used fields through byte 112 must be file-backed; a trailing declared zero-fill @@ -354,7 +378,8 @@ the result has no rows, and `generic_method_pointers_field`, `generic_method_pointers_count` and `generic_method_table_stride` are `null`. A table of a few rows can still leave two layouts agreeing, which fails as ambiguous rather than picking one. Metadata versions 24 and 39 and builds -without a static CodeRegistration (`code_gen_modules_field` `null`) fail as +without a static CodeRegistration (`code_gen_modules_field` `null`) or +MetadataRegistration (`types_field` `null`) fail as `generic-methods-unsupported`: for version 24 the type-definition generic container offset and the generic class layout have not been checked against a real build, so it is refused rather than guessed. @@ -419,10 +444,10 @@ The builds checked are listed after the module search rules above. | `code-registration-not-found` | input error | No unique module table or executable-byte array candidate was resolved. | | `invalid-method-token` | input error | A token is outside its module's method-pointer rows, or a MethodDef row is reused within one image. | | `invalid-method-pointer` | input error | A slot is unavailable or its nonzero address lacks unambiguous executable file coverage. | -| `metadata-registration-not-found` | input error | No single MetadataRegistration matches the type-definition count. | +| `metadata-registration-not-found` | input error | No single MetadataRegistration or code-loaded type-table candidate was resolved. | | `invalid-type` | input error | A type index or `Il2CppType` record is out of range, unmapped or malformed. | | `unsupported-type-kind` | unsupported | A valid type kind has no implemented rendering. | -| `generic-methods-unsupported` | unsupported | Generic instances were requested for metadata version 24 or 39 or a build without a static CodeRegistration. | +| `generic-methods-unsupported` | unsupported | Generic instances were requested for metadata version 24 or 39, or a build lacks either static registration. | | `generic-method-registration-not-found` | input error | No CodeRegistration layout bounds every genericMethodTable row, or the generic tables are unreadable. | | `ambiguous-generic-method-registration` | input error | More than one CodeRegistration layout bounds every genericMethodTable row. | diff --git a/src/application/il2cpp/Il2cppMetadataService.test.ts b/src/application/il2cpp/Il2cppMetadataService.test.ts index 6ec65a5704..696fcc158f 100644 --- a/src/application/il2cpp/Il2cppMetadataService.test.ts +++ b/src/application/il2cpp/Il2cppMetadataService.test.ts @@ -261,13 +261,17 @@ it("closes both ports and keeps combined cleanup incomplete until both owners re metadata.resolveCleanup(); expect(failure.cleanupIncomplete).toBe(false); }); -it.each([false, true])( - "classifies native association confidence (inferred: %s)", - async (inferred) => { +it.each([ + ["static", null, "derived"], + ["module array", "code_gen_modules_field", "inferred"], + ["type table", "types_field", "inferred"], +] as const)( + "classifies native association confidence (%s)", + async (_, inferred, confidence) => { const value = linkedReport(); - if (inferred) value.binary!.code_gen_modules_field = null; + if (inferred !== null) value.binary![inferred] = null; const result = await serviceReturning(value).inspect(selectedPair); if (!result.ok) throw result.error; - expect(result.value.confidence).toBe(inferred ? "inferred" : "derived"); + expect(result.value.confidence).toBe(confidence); }, ); diff --git a/src/application/il2cpp/Il2cppMetadataService.ts b/src/application/il2cpp/Il2cppMetadataService.ts index 59ecc52f57..76cdae05f2 100644 --- a/src/application/il2cpp/Il2cppMetadataService.ts +++ b/src/application/il2cpp/Il2cppMetadataService.ts @@ -240,7 +240,8 @@ export class Il2cppMetadataService { confidence: value.binary === null ? "observed" - : value.binary.code_gen_modules_field === null + : value.binary.code_gen_modules_field === null || + value.binary.types_field === null ? "inferred" : "derived", limitations: value.limitations, diff --git a/src/contracts/il2cpp/il2cppToolContracts.ts b/src/contracts/il2cpp/il2cppToolContracts.ts index 975fa74df1..d0da88d605 100644 --- a/src/contracts/il2cpp/il2cppToolContracts.ts +++ b/src/contracts/il2cpp/il2cppToolContracts.ts @@ -16,7 +16,7 @@ export const IL2CPP_TOOL_CONTRACTS = [ kind: "artifact-provider", requiresActiveTarget: true, description: - "Inspect local Unity IL2CPP metadata (24/27/29/31/39), optionally linking a caller-selected 64-bit little-endian Mach-O, PE or ELF binary. Static reading only. Default summary retains complete Evidence; use parent_evidence_id with inspect_analysis_view kind il2cpp-metadata for named collections, indices, fields and paging. Parameters accept method_indices; method_addresses accept addresses and states. Stable row indices are scoped to the metadata digest; native pages also identify the binary. Native states distinguish executable code, observed zero slots (no-body), and unknown owners/relocations. Shared addresses preserve all methods. code_gen_modules pages accept image_indices; summaries omit that table. Type names are C#-like; version27 retains indices with unknown names. generic_instances adds genericMethodTable rows with Il2CppDumper-style names (versions 27/29/31, static CodeRegistration); generic_method_instances pages accept method_indices, addresses and states; summaries count issues, paged as generic_method_issues. Unrequested native counts are null. Static-table links are derived; the x86-64 executable-byte fallback is inferred and does not verify instruction boundaries. Input guards:256MiB metadata/1GiB binary;64MiB encoded report; one active inspection. Unsupported layouts, pointer encodings and resource limits have typed reasons. Full Evidence remains available with detail complete. Neither static linking nor structural agreement proves runtime behavior or a same-build pair.", + "Inspect local Unity IL2CPP metadata (24/27/29/31/39), optionally linking a caller-selected 64-bit little-endian Mach-O, PE or ELF binary. Static reading only. Default summary retains complete Evidence; use parent_evidence_id with inspect_analysis_view kind il2cpp-metadata for named collections, indices, fields and paging. Parameters accept method_indices; method_addresses accept addresses and states. Stable row indices are scoped to the metadata digest; native pages also identify the binary. Native states distinguish executable code, observed zero slots (no-body), and unknown owners/relocations. Shared addresses preserve all methods. code_gen_modules pages accept image_indices; summaries omit that table. Type names are C#-like; version27 retains indices with unknown names. generic_instances adds genericMethodTable rows with Il2CppDumper-style names (versions 27/29/31, static CodeRegistration and MetadataRegistration); generic_method_instances pages accept method_indices, addresses and states; summaries count issues, paged as generic_method_issues. Unrequested native counts are null. Static-table links are derived; the x86-64/arm64 executable-byte fallback is inferred and does not verify instruction boundaries. Input guards:256MiB metadata/1GiB binary;64MiB encoded report; one active inspection. Unsupported layouts, pointer encodings and resource limits have typed reasons. Full Evidence remains available with detail complete. Neither static linking nor structural agreement proves runtime behavior or a same-build pair.", inputSchema: inspectIl2cppMetadataInputSchema, outputSchema: evidenceResultOf( z.union([il2cppMetadataSchema, il2cppMetadataSummaryViewResultSchema]), diff --git a/src/domain/analysisView/il2cppMetadataView.test.ts b/src/domain/analysisView/il2cppMetadataView.test.ts index 2aa3263ee2..1ac3c4f01f 100644 --- a/src/domain/analysisView/il2cppMetadataView.test.ts +++ b/src/domain/analysisView/il2cppMetadataView.test.ts @@ -554,6 +554,24 @@ it("keeps native summaries bounded while paging selected module facts", () => { }); }); +it.each([ + [ + "code_gen_modules_field", + "Module-array association", + "Type-table association", + ], + ["types_field", "Type-table association", "Module-array association"], +] as const)( + "keeps an inferred %s as its own unknown", + (field, present, absent) => { + const value = linkedReport(); + value.binary![field] = null; + const unknowns = summarize(parent(value, false)).unknowns.map(String); + expect(unknowns.filter((line) => line.startsWith(present))).toHaveLength(1); + expect(unknowns.some((line) => line.startsWith(absent))).toBe(false); + }, +); + const genericLinked = (): Il2cppMetadata => { const linked = linkedReport(); return { diff --git a/src/domain/analysisView/il2cppMetadataView.ts b/src/domain/analysisView/il2cppMetadataView.ts index 7f1db7831e..e18cbeebc2 100644 --- a/src/domain/analysisView/il2cppMetadataView.ts +++ b/src/domain/analysisView/il2cppMetadataView.ts @@ -606,6 +606,11 @@ export const projectIl2cppMetadataView = ( "Module-array association is inferred from executable byte patterns; instruction boundaries and runtime use remain unverified.", ] : []), + ...(report.binary != null && report.binary.types_field === null + ? [ + "Type-table association and types_count are inferred from executable byte patterns and Il2CppType-shaped records; instruction boundaries and runtime use remain unverified.", + ] + : []), ...[ ...il2cppMetadataMemberCollections, ...il2cppMetadataBinaryCollections, diff --git a/src/domain/il2cpp/il2cppMetadata.ts b/src/domain/il2cpp/il2cppMetadata.ts index f589877bfe..5d21a659f4 100644 --- a/src/domain/il2cpp/il2cppMetadata.ts +++ b/src/domain/il2cpp/il2cppMetadata.ts @@ -288,8 +288,12 @@ export const il2cppBinaryLinkSchema = z.strictObject({ /** The code-generation module pointer array. */ code_gen_modules_array: il2cppAddressSchema, code_gen_modules: z.array(il2cppCodeGenModuleSchema), - /** MetadataRegistration's typesCount field, followed by the types pointer. */ - types_field: il2cppAddressSchema, + /** + * MetadataRegistration's typesCount field, followed by the types pointer; + * null when no static registration had the counts and executable byte-pattern + * candidates identified the type table, whose run length is then types_count. + */ + types_field: il2cppAddressSchema.nullable(), types_count: count, /** Null unless generic_instances was requested. */ generic_methods: il2cppGenericMethodLinkSchema.nullable(), diff --git a/src/il2cpp/Il2cppBinaryFile.ts b/src/il2cpp/Il2cppBinaryFile.ts index da738ce63b..0bcd18d74c 100644 --- a/src/il2cpp/Il2cppBinaryFile.ts +++ b/src/il2cpp/Il2cppBinaryFile.ts @@ -45,8 +45,13 @@ export interface Il2cppBinaryLink { * slot is zero, "unknown" when its owner or relocated pointer is unresolved. */ readonly method_addresses: Il2cppMethodAddress[]; - /** Address of MetadataRegistration's typesCount field. */ - readonly types_field: bigint; + /** + * Address of MetadataRegistration's typesCount field, or null when no static + * registration matched and an executable byte-pattern candidate identified + * the type table; types_count is then the inferred run length, at least + * every referenced type index but not an observed typesCount. + */ + readonly types_field: bigint | null; readonly types_count: number; /** Il2CppType table index and display name per method return, field and parameter. */ readonly return_types: Il2cppTypeReference[]; @@ -170,6 +175,7 @@ export function* linkIl2cppBinarySteps( const registration = yield* findMetadataRegistration( image, metadata.types.length, + yield* referencedTypeCount(facts), ); const typeName = createTypeNamer( image, @@ -222,7 +228,7 @@ export function* linkIl2cppBinarySteps( facts, registration_.field, // findMetadataRegistration reports typesCount, word 6. - registration.field - 48n, + registration.field === null ? null : registration.field - 48n, options, ) : null; @@ -243,6 +249,23 @@ export function* linkIl2cppBinarySteps( }; } +/** One past the highest type index any return, field or parameter references. */ +function* referencedTypeCount( + facts: Il2cppMetadataLinkFacts, +): Generator { + let count = 0; + for (const types of [ + facts.returnTypes, + facts.fieldTypes, + facts.parameterTypes, + ]) + for (let index = 0; index < types.length; index++) { + if (index % LINK_BATCH === 0) yield; + count = Math.max(count, types[index]! + 1); + } + return count; +} + /** Rows between cooperative yields while linking. */ const LINK_BATCH = 256; diff --git a/src/il2cpp/Il2cppBinaryGenerics.ts b/src/il2cpp/Il2cppBinaryGenerics.ts index 7b286bb31d..bf3b2582da 100644 --- a/src/il2cpp/Il2cppBinaryGenerics.ts +++ b/src/il2cpp/Il2cppBinaryGenerics.ts @@ -103,7 +103,8 @@ export function* linkGenericMethodsSteps( metadata: Il2cppMetadataDecoded, facts: Il2cppMetadataLinkFacts, modulesField: bigint | null, - metadataRegistration: bigint, + /** Null when the type table was inferred without a static MetadataRegistration. */ + metadataRegistration: bigint | null, options: Il2cppBinaryReadOptions, ): Generator { const version = metadata.metadata_version; @@ -113,6 +114,11 @@ export function* linkGenericMethodsSteps( "generic-methods-unsupported", "this binary keeps no static CodeRegistration, so genericMethodPointers cannot be located", ); + if (metadataRegistration === null) + throw new Il2cppBinaryFormatFailure( + "generic-methods-unsupported", + "this binary keeps no static MetadataRegistration, so genericMethodTable cannot be located", + ); const word = (address: bigint) => readPointer(image, address); const count = (address: bigint) => { const value = word(address); diff --git a/src/il2cpp/Il2cppBinaryImage.ts b/src/il2cpp/Il2cppBinaryImage.ts index 9716549f2c..e8ff9fd516 100644 --- a/src/il2cpp/Il2cppBinaryImage.ts +++ b/src/il2cpp/Il2cppBinaryImage.ts @@ -6,9 +6,11 @@ import { binaryMetadataArchitecture } from "../domain/binaryMetadataArchitecture import { BinaryFormatFailure, BinaryResourceFailure, + createFileMappingIndex, mappedFileOffset, isZeroFilledSpan, type BinaryContainer, + type FileMappingIndex, } from "../native/metadata/BinaryContainer.js"; import { readBinaryContainer } from "../native/metadata/readBinaryContainer.js"; import { @@ -74,6 +76,8 @@ const selectImage = ( export interface NativeImage { readonly bytes: Buffer; readonly container: BinaryContainer; + /** One image's immutable load mappings, shared by stored and loader-aware reads. */ + readonly fileMappings: FileMappingIndex; readonly imageBase: bigint; readonly unresolvedPointerSlots: ReadonlyMap; readonly zeroFillPointers: ReadonlyMap; @@ -190,6 +194,10 @@ export function* openImageSteps( return { bytes: image, container, + fileMappings: createFileMappingIndex( + container.mappings, + container.zero_fills, + ), imageBase, unresolvedPointerSlots: relocated?.unresolved ?? new Map(), zeroFillPointers: relocated?.zeroFillPointers ?? new Map(), @@ -200,12 +208,7 @@ export function* openImageSteps( /** Stored bytes only; loader interpretation belongs to readSlot. */ const readStoredWord = (image: NativeImage, address: bigint): bigint | null => { try { - const offset = mappedFileOffset( - image.container.mappings, - address, - 8, - image.container.zero_fills, - ); + const offset = image.fileMappings.offset(address, 8); return image.bytes.readBigUInt64LE(offset); } catch (cause) { if (cause instanceof BinaryFormatFailure) return null; @@ -226,14 +229,7 @@ export const readUInt32 = ( const virtual = image.zeroFillPointers.get(address); if (virtual !== undefined) return Number(virtual & 0xffffffffn); try { - return image.bytes.readUInt32LE( - mappedFileOffset( - image.container.mappings, - address, - 4, - image.container.zero_fills, - ), - ); + return image.bytes.readUInt32LE(image.fileMappings.offset(address, 4)); } catch (cause) { if (cause instanceof BinaryFormatFailure) return null; throw cause; @@ -245,12 +241,7 @@ export const hexAddress = (value: bigint) => `0x${value.toString(16)}`; export const isMapped = (image: NativeImage, address: bigint, size: number) => { try { - mappedFileOffset( - image.container.mappings, - address, - size, - image.container.zero_fills, - ); + image.fileMappings.offset(address, size); return true; } catch (cause) { if (cause instanceof BinaryFormatFailure) return false; @@ -302,12 +293,7 @@ export const hasPointerArrayCoverage = ( if (fileEnd === cursor) { if (!isZeroFilled(image, cursor, size)) return false; } else { - mappedFileOffset( - image.container.mappings, - cursor, - size, - image.container.zero_fills, - ); + image.fileMappings.offset(cursor, size); } } catch (cause) { if (cause instanceof BinaryFormatFailure) return false; @@ -321,12 +307,7 @@ export const hasPointerArrayCoverage = ( /** Both executable coverage and the unique full loader mapping must agree. */ export const isExecutable = (image: NativeImage, address: bigint): boolean => { try { - const offset = mappedFileOffset( - image.container.mappings, - address, - 1, - image.container.zero_fills, - ); + const offset = image.fileMappings.offset(address, 1); return ( offset === mappedFileOffset(image.container.executable_mappings ?? [], address, 1) @@ -359,8 +340,12 @@ export const readSlot = ( const virtual = image.zeroFillPointers.get(address); let raw: bigint | null; if (virtual !== undefined) raw = virtual; - else if (isZeroFilled(image, address, 8)) raw = 0n; - else raw = readStoredWord(image, address); + else { + // A successful indexed read already excludes zero-fill aliases. Only a + // non-file-backed slot needs the zero-fill ownership walk. + raw = readStoredWord(image, address); + if (raw === null && isZeroFilled(image, address, 8)) raw = 0n; + } if (raw === null) return null; if (image.machoPointerFixups !== undefined) { const resolution = image.machoPointerFixups.decode(address, raw); diff --git a/src/il2cpp/Il2cppBinaryRegistration.ts b/src/il2cpp/Il2cppBinaryRegistration.ts index 02a98caa68..ed14a452e3 100644 --- a/src/il2cpp/Il2cppBinaryRegistration.ts +++ b/src/il2cpp/Il2cppBinaryRegistration.ts @@ -7,10 +7,16 @@ import { isMapped, hasPointerArrayCoverage, readPointer, + readSlot, readUInt32, type NativeImage, } from "./Il2cppBinaryImage.js"; import type { Il2cppMetadataDecoded } from "./Il2cppMetadataFile.js"; +import { + IL2CPP_TYPE_KIND_MAX, + il2cppTypeKind, + isDefinitionTypeKind, +} from "./Il2cppBinaryTypes.js"; import type { Il2cppCodeGenModule } from "./Il2cppBinaryFile.js"; // Bound the retained name-address index independently of binary size. No // repeated name is evidence of ambiguous module ownership by itself. @@ -245,7 +251,11 @@ export function* findCodeRegistration( } } } - if (matches.length === 0 && image.container.architecture === "amd64") { + const architecture = image.container.architecture; + if ( + matches.length === 0 && + (architecture === "amd64" || architecture === "arm64") + ) { const array = yield* findModuleArrayByCode(image, expected); if (array !== null) return { field: null, array }; } @@ -258,12 +268,16 @@ export function* findCodeRegistration( } /** - * Some x86-64 builds (a Unity 2023.2 Linux GameAssembly.so) keep no static - * CodeRegistration: the compiler folded the module array and its count into - * the one loop that walks it, so no data word points at the array. Find every + * Some builds keep no static CodeRegistration: the compiler folded the module + * array and its count into the one loop that walks it, so no data word points + * at the array. Seen on x86-64 (a Unity 2023.2 Linux GameAssembly.so) and on + * arm64 (a Unity 2022.3 macOS Master/LTO GameAssembly.dylib). Find every * pointer array that lists exactly the located modules, then require exactly - * one of them to be the target of a RIP-relative `lea`/`mov` (REX.W 8D/8B with - * a disp32 operand). Anything else stays code-registration-not-found. + * one of them to be the target of an address-forming instruction in an + * executable range: on x86-64 a RIP-relative `lea`/`mov` (REX.W 8D/8B with a + * disp32 operand), on arm64 an `adrp` immediately followed by an `add` of its + * page offset to the same register. Anything else stays + * code-registration-not-found. */ const MODULE_ARRAY_CANDIDATES = 64; @@ -303,6 +317,17 @@ function* findModuleArrayByCode( } } if (arrays.size === 0) return null; + const referenced = + image.container.architecture === "arm64" + ? yield* arm64PageReferences(image, arrays) + : yield* ripRelativeReferences(image, arrays); + return referenced.size === 1 ? [...referenced][0]! : null; +} + +function* ripRelativeReferences( + image: NativeImage, + arrays: ReadonlySet, +): Generator> { const referenced = new Set(); for (const mapping of image.container.executable_mappings ?? []) { const bytes = image.bytes; @@ -319,7 +344,44 @@ function* findModuleArrayByCode( if (arrays.has(target)) referenced.add(target); } } - return referenced.size === 1 ? [...referenced][0]! : null; + return referenced; +} + +/** + * arm64 instructions are 4-byte aligned words, so only aligned addresses are + * decoded. `adrp xd, page` forms the 4 KiB page of pc plus a signed 21-bit + * page delta; the next word must be `add xd, xd, #imm12` (64-bit, unshifted), + * which adds the offset within that page. Pairs split by other instructions or + * loads through `ldr` are not followed. + */ +function* arm64PageReferences( + image: NativeImage, + arrays: ReadonlySet, +): Generator> { + const referenced = new Set(); + const bytes = image.bytes; + for (const mapping of image.container.executable_mappings ?? []) { + const start = Number((4n - (mapping.address % 4n)) % 4n); + for (let at = start; at + 8 <= mapping.size; at += 4) { + if ((at & 0xffff) < 4) yield; + const adrp = bytes.readUInt32LE(mapping.offset + at); + if ((adrp & 0x9f000000) >>> 0 !== 0x90000000) continue; + const add = bytes.readUInt32LE(mapping.offset + at + 4); + if ((add & 0xffc00000) >>> 0 !== 0x91000000) continue; + const register = adrp & 0x1f; + // Register 31 is xzr for ADRP and sp for ADD, so no address is formed. + if (register === 31) continue; + if ((add & 0x1f) !== register || ((add >>> 5) & 0x1f) !== register) + continue; + const pages = + (((((adrp >>> 5) & 0x7ffff) << 2) | ((adrp >>> 29) & 3)) << 11) >> 11; + const pc = mapping.address + BigInt(at); + const target = + (pc & ~0xfffn) + (BigInt(pages) << 12n) + BigInt((add >>> 10) & 0xfff); + if (arrays.has(target)) referenced.add(target); + } + } + return referenced; } /** @@ -327,20 +389,26 @@ function* findModuleArrayByCode( * fieldOffsetsCount and typeDefinitionsSizesCount (words 10 and 12) both equal * the metadata type-definition count and are followed by mapped pointers. * typesCount and types are words 6 and 7. Counts are int32 in 8-byte slots. + * Linking reads only the type table, so when no static registration matches on + * x86-64 or arm64 the table is inferred from code instead (field null). */ export function* findMetadataRegistration( image: NativeImage, typeDefinitions: number, + requiredTypes: number, ): Generator< void, { - readonly field: bigint; + readonly field: bigint | null; readonly count: number; readonly types: bigint; } > { const matches: { field: bigint; count: number; types: bigint }[] = []; const matchedFields = new Set(); + // A record whose two counts match is static evidence even when its other + // fields are unusable; code is then not asked to stand in for it. + let countsMatched = false; const low = (offset: number) => image.bytes.readUInt32LE(offset); for (const mapping of image.container.mappings) { const start = Number((8n - (mapping.address % 8n)) % 8n); @@ -355,6 +423,7 @@ export function* findMetadataRegistration( if (!hasPointerArrayCoverage(image, address, 16)) continue; if (readUInt32(image, address + 80n) !== typeDefinitions) continue; if (readUInt32(image, address + 96n) !== typeDefinitions) continue; + countsMatched = true; const count = readUInt32(image, address + 48n); const types = readPointer(image, address + 56n); const fieldOffsets = readPointer(image, address + 88n); @@ -387,10 +456,181 @@ export function* findMetadataRegistration( }); } } + const architecture = image.container.architecture; + let inferred = false; + if ( + !countsMatched && + requiredTypes > 0 && + (architecture === "amd64" || architecture === "arm64") + ) { + inferred = true; + const table = yield* findTypeTableByCode( + image, + typeDefinitions, + requiredTypes, + ); + if (table !== null) return { field: null, ...table }; + } if (matches.length !== 1) throw new Il2cppBinaryFormatFailure( "metadata-registration-not-found", - `${matches.length} MetadataRegistration candidates count ${typeDefinitions} type definitions`, + countsMatched && matches.length === 0 + ? `a MetadataRegistration counts ${typeDefinitions} type definitions, but its type table, field offsets or type sizes are not mapped` + : inferred + ? `no MetadataRegistration counts ${typeDefinitions} type definitions, and no single code-loaded run of Il2CppType pointers covers every referenced type index` + : `${matches.length} MetadataRegistration candidates count ${typeDefinitions} type definitions`, ); return matches[0]!; } + +/** + * An inlined MetadataRegistration leaves the Il2CppType pointer table loaded + * directly by code. Candidates are maximal runs of pointers to Il2CppType-shaped + * records (a defined Il2CppTypeEnum kind; class and value-type data below the + * metadata type-definition count) long enough for every type index the + * metadata references. Exactly one such run may exist in the image, it must + * end inside its mapping, and code must load its start with the same + * instruction forms as the module-array fallback; anything else fails closed. + * The run length becomes the count: never below what the metadata needs, but + * not an observed typesCount. + */ +/** Il2CppTypeEnum values 0x01 to 0x1e; 0x17 and 0x1a are unassigned. */ +const isDefinedTypeKind = (kind: number) => + kind >= 1 && kind <= IL2CPP_TYPE_KIND_MAX && kind !== 0x17 && kind !== 0x1a; + +function* findTypeTableByCode( + image: NativeImage, + typeDefinitions: number, + requiredTypes: number, +): Generator { + const { bytes, container } = image; + // Nomination and loader-aware confirmation share the image's mapping index. + // Most words are not record addresses; reject those without the throwing + // lookup. Starts are sorted, and reach[i] is the furthest end among the + // first i + 1 mappings, so one binary search finds a possible container. + const spans = container.mappings + .filter((mapping) => mapping.size > 0) + .map((mapping) => ({ + start: mapping.address, + end: mapping.address + BigInt(mapping.size), + })) + .sort((left, right) => + left.start < right.start ? -1 : left.start > right.start ? 1 : 0, + ); + const reach: bigint[] = []; + for (const [index, span] of spans.entries()) + reach.push( + index === 0 || span.end > reach[index - 1]! + ? span.end + : reach[index - 1]!, + ); + const maybeMapped = (address: bigint) => { + let lowIndex = 0; + let highIndex = spans.length - 1; + let found = -1; + while (lowIndex <= highIndex) { + const middle = (lowIndex + highIndex) >> 1; + if (spans[middle]!.start <= address) { + found = middle; + lowIndex = middle + 1; + } else highIndex = middle - 1; + } + return found >= 0 && reach[found]! >= address + 12n; + }; + const recordOffset = (address: bigint) => { + if (!maybeMapped(address)) return -1; + try { + return image.fileMappings.offset(address, 12); + } catch (cause) { + // Unmapped, zero-filled or ambiguously mapped records are not types. + if (cause instanceof BinaryFormatFailure) return -1; + throw cause; + } + }; + const isType = (address: bigint) => { + // Records hold an 8-byte pointer first; zero and unaligned words are not types. + if (address === 0n || (address & 7n) !== 0n) return false; + const offset = recordOffset(address); + if (offset < 0) return false; + const kind = il2cppTypeKind(bytes.readUInt32LE(offset + 8)); + if (!isDefinedTypeKind(kind)) return false; + if (!isDefinitionTypeKind(kind)) return true; + return bytes.readBigUInt64LE(offset) < BigInt(typeDefinitions); + }; + // A run that touches either end of its mapping may continue in a + // neighbouring one, so its length and start are unknown; it still counts as + // a candidate but cannot be chosen. + const runs = new Map(); + // Stored words only nominate slots: a slot the loader binds or leaves + // unresolved has no known value, so a run it bounds has an unknown extent + // whatever its placeholder holds. + // The same holds for a record's kind word and its definition index. + const recordState = (target: bigint): "type" | "other" | "unknown" => { + if (recordOffset(target) < 0) return "other"; + const attributes = readUInt32(image, target + 8n); + if (attributes === "unknown") return "unknown"; + if (attributes === null) return "other"; + const kind = il2cppTypeKind(attributes); + if (!isDefinedTypeKind(kind)) return "other"; + if (!isDefinitionTypeKind(kind)) return "type"; + const data = readSlot(image, target); + if (data === "unknown") return "unknown"; + return typeof data === "bigint" && data < BigInt(typeDefinitions) + ? "type" + : "other"; + }; + // A word whose slot or record the loader leaves unknown bounds a run with + // an unknown extent. + const unresolved = (address: bigint) => { + const slot = readSlot(image, address); + return ( + slot === "unknown" || + (typeof slot === "bigint" && recordState(slot) === "unknown") + ); + }; + for (const mapping of container.mappings) { + const start = Number((8n - (mapping.address % 8n)) % 8n); + let runStart = -1; + let startUnknown = false; + for (let at = start; at <= mapping.size; at += 8) { + if ((at & 0xffff) < 8) yield; + const address = mapping.address + BigInt(at); + let endUnknown = at + 8 > mapping.size; + if (!endUnknown) { + const raw = bytes.readBigUInt64LE(mapping.offset + at); + if (isType(raw)) { + const slot = readSlot(image, address); + const state = slot === raw ? recordState(slot) : "other"; + if (state === "type") { + if (runStart < 0) { + runStart = at; + startUnknown = at === start || unresolved(address - 8n); + } + continue; + } + endUnknown = slot === "unknown" || state === "unknown"; + } else if (runStart >= 0) endUnknown = unresolved(address); + } + if (runStart >= 0 && (at - runStart) / 8 >= requiredTypes) { + runs.set( + mapping.address + BigInt(runStart), + endUnknown || startUnknown ? null : (at - runStart) / 8, + ); + // Another long-enough run could be the real table while code loads + // only this one, so a second candidate fails closed at once. + if (runs.size > 1) return null; + } + runStart = -1; + } + } + if (runs.size !== 1) return null; + const [start, count] = [...runs][0]!; + if (count === null) return null; + const starts = new Set([start]); + const referenced = + container.architecture === "arm64" + ? yield* arm64PageReferences(image, starts) + : yield* ripRelativeReferences(image, starts); + if (!referenced.has(start)) return null; + return { count, types: start }; +} diff --git a/src/il2cpp/Il2cppBinaryTypes.ts b/src/il2cpp/Il2cppBinaryTypes.ts index c29b05d9dc..57af891a6f 100644 --- a/src/il2cpp/Il2cppBinaryTypes.ts +++ b/src/il2cpp/Il2cppBinaryTypes.ts @@ -9,6 +9,16 @@ import type { Il2cppMetadataDecoded, Il2cppMetadataLinkFacts, } from "./Il2cppMetadataFile.js"; + +/** Il2CppType's type enum: bits 16-23 of the word after its 8-byte data. */ +export const il2cppTypeKind = (attributes: number): number => + (attributes >>> 16) & 0xff; +/** Largest Il2CppTypeEnum value read as a type kind (IL2CPP_TYPE_MVAR). */ +export const IL2CPP_TYPE_KIND_MAX = 0x1e; +/** Class and value-type kinds, whose data is a TypeDefinitionIndex. */ +export const isDefinitionTypeKind = (kind: number): boolean => + kind === 0x11 || kind === 0x12; + const PRIMITIVES = new Map([ [0x01, "void"], [0x02, "bool"], @@ -127,12 +137,12 @@ const createTypeRenderer = ( if (tick()) yield; const data = pointer(address, "Il2CppType data"); const bits = u32(address + 8n, "Il2CppType attributes"); - const kind = (bits >>> 16) & 0xff; + const kind = il2cppTypeKind(bits); const index = Number(BigInt.asIntN(32, data & 0xffffffffn)); let result: string; const primitive = rules.primitives.get(kind); if (primitive !== undefined) result = primitive; - else if (kind === 0x11 || kind === 0x12) + else if (isDefinitionTypeKind(kind)) result = yield* rules.definition(index); else if (kind === 0x0f || kind === 0x1d) result = join([ diff --git a/src/il2cpp/Il2cppMetadataProvider.ts b/src/il2cpp/Il2cppMetadataProvider.ts index 8818b6c6de..ec20e8cf85 100644 --- a/src/il2cpp/Il2cppMetadataProvider.ts +++ b/src/il2cpp/Il2cppMetadataProvider.ts @@ -67,6 +67,11 @@ const INPUT_BYTES = 256 * 1024 * 1024; const BINARY_BYTES = 1024 * 1024 * 1024; // A 12.9 MB Unity 6000.4 file with 112,232 methods serializes to 7.8 MB. const OUTPUT_BYTES = 64 * 1024 * 1024; +/** Instruction form the module-array fallback matched, per architecture. */ +const inferredReferenceForm = (architecture: string) => + architecture === "arm64" + ? "an adjacent ADRP+ADD pair" + : "RIP-relative LEA/MOV"; const failureForPrimary = ( facts: ArtifactFilePrimaryFacts, path: string, @@ -229,7 +234,7 @@ const binaryReport = ( address: hex(module.address), method_pointer_count: module.method_pointer_count, })), - types_field: hex(link.types_field), + types_field: link.types_field === null ? null : hex(link.types_field), types_count: link.types_count, generic_methods: link.generic_methods === null @@ -587,7 +592,12 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { ...(linked.binary !== null && linked.binary.code_gen_modules_field === null ? [ - "The static CodeRegistration pair was not resolved. The module array is inferred from executable bytes consistent with RIP-relative LEA/MOV referencing one array candidate; instruction boundaries and runtime use were not verified.", + `The static CodeRegistration pair was not resolved. The module array is inferred from executable bytes consistent with ${inferredReferenceForm(linked.binary.architecture)} referencing one array candidate; instruction boundaries and runtime use were not verified.`, + ] + : []), + ...(linked.binary !== null && linked.binary.types_field === null + ? [ + `No static MetadataRegistration had the type-definition counts. The type table is inferred from executable bytes consistent with ${inferredReferenceForm(linked.binary.architecture)} referencing the one run of Il2CppType-shaped pointers long enough for every referenced type index; types_count is that run's length, at least every referenced type index but not an observed typesCount, and instruction boundaries and runtime use were not verified.`, ] : []), ...(binaryPath === undefined diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index 3f754a3c64..224121527d 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -50,6 +50,14 @@ const expectedAddress = (value: bigint | "no-body" | "unknown") => const expectedAddresses = FIXTURE_METHOD_ADDRESSES.map(expectedAddress); const link = (bytes: Buffer, architecture?: string) => linkIl2cppBinary(bytes, metadata, facts, architecture); +// A type-table fallback needs the metadata to reference most of the table, as +// real builds do; referencing type 3 requires all four fixture entries. +const wholeTable = { + ...facts, + returnTypes: Int32Array.from(facts.returnTypes, () => 3), +}; +const linkWholeTable = (bytes: Buffer) => + linkIl2cppBinary(bytes, metadata, wholeTable); const failure = (bytes: Buffer, architecture?: string) => { try { link(bytes, architecture); @@ -430,6 +438,389 @@ it.each([ ).toBe("code-registration-not-found"); }); +it("finds an arm64 module array through the ADRP+ADD pair that loads it", () => { + const linked = link( + createIl2cppBinaryFixture({ + codeRegistration: false, + adrpAdd: [[0xc00, 0x500]], + }), + ); + expect(linked.code_gen_modules_field).toBeNull(); + expect(linked.code_gen_modules_array).toBe(FIXTURE_BINARY_BASE + 0x500n); + expect(linked.method_addresses).toEqual(expectedAddresses); +}); + +it.each([ + ["no instruction pair loads the array", {}], + ["the pair starts off a 4-byte instruction boundary", { at: 0xc02 }], + ["two module-array candidates are each loaded", { second: true }], + ["the pair lies outside executable ranges", { protection: 3 }], +] as const)( + "keeps arm64 code-registration-not-found when %s", + (_, variant: { at?: number; second?: boolean; protection?: number }) => { + const bytes = createIl2cppBinaryFixture({ + codeRegistration: false, + adrpAdd: + Object.keys(variant).length === 0 + ? [] + : [ + [variant.at ?? 0xc00, 0x500], + ...(variant.second === true ? [[0xc10, 0x520] as const] : []), + ], + }); + if (variant.second === true) bytes.copy(bytes, 0x520, 0x500, 0x510); + if (variant.protection !== undefined) + bytes.writeUInt32LE(variant.protection, 92); + expect(failure(bytes).reason).toBe("code-registration-not-found"); + }, +); + +it("finds an arm64 type table through the ADRP+ADD pair that loads it", () => { + const linked = linkWholeTable( + createIl2cppBinaryFixture({ + metadataRegistration: false, + adrpAdd: [[0xc20, 0x900]], + }), + ); + const reference = linkWholeTable(createIl2cppBinaryFixture()); + expect(linked.types_field).toBeNull(); + expect(linked.types_count).toBe(4); + expect(linked.return_types).toEqual(reference.return_types); + expect(linked.field_types).toEqual(reference.field_types); + expect(linked.parameter_types).toEqual(reference.parameter_types); +}); + +it("finds an x86-64 type table through the instruction that loads it", () => { + const image = createIl2cppBinaryFixture({ metadataRegistration: false }); + // lea rcx, [rip + disp32] at 0xc40, loading the type table at 0x900. + image.set([0x48, 0x8d, 0x0d], 0xc40); + image.writeInt32LE(0x900 - (0xc40 + 7), 0xc43); + const linked = linkWholeTable(createElfFixture(image, { machine: 62 })); + expect(linked.types_field).toBeNull(); + expect(linked.types_count).toBe(4); + expect(linked.return_types).toEqual( + linkWholeTable( + createElfFixture(createIl2cppBinaryFixture(), { machine: 62 }), + ).return_types, + ); +}); + +it.each([ + ["the pair loads an address inside the run", { target: 0x908 }], + [ + "a type-shaped word extends the run before the loaded address", + { before: true }, + ], + ["two runs are each loaded", { second: true }], + [ + "the loaded run is shorter than the referenced type indices", + { short: true }, + ], + ["a record's kind is past the Il2CppTypeEnum range", { kind: 0x1f }], + [ + "a value-type record names a type definition past the metadata", + { definition: 4n }, + ], +] as const)( + "keeps metadata-registration-not-found when %s", + ( + _, + variant: { + target?: number; + before?: boolean; + second?: boolean; + short?: boolean; + kind?: number; + definition?: bigint; + }, + ) => { + const bytes = createIl2cppBinaryFixture({ + metadataRegistration: false, + adrpAdd: [ + [0xc20, variant.target ?? 0x900], + ...(variant.second === true ? [[0xc30, 0xb00] as const] : []), + ], + }); + if (variant.second === true) bytes.copy(bytes, 0xb00, 0x900, 0x920); + // The int record (type 3) is a valid Il2CppType, so the run starts at 0x8f8. + if (variant.before === true) + bytes.writeBigUInt64LE(FIXTURE_BINARY_BASE + 0x9b0n, 0x8f8); + // Kind 0 ends the run after two entries; the facts below reference type 3. + if (variant.short === true) bytes.writeUInt32LE(0, 0x980 + 2 * 16 + 8); + // Either record breaks the four-entry run into pieces shorter than four. + if (variant.kind !== undefined) + bytes.writeUInt32LE(variant.kind << 16, 0x980 + 16 + 8); + if (variant.definition !== undefined) + bytes.writeBigUInt64LE(variant.definition, 0x980 + 2 * 16); + // Facts referencing type 3 require a four-entry run. + const required = + variant.short === true || + variant.kind !== undefined || + variant.definition !== undefined + ? { ...facts, returnTypes: Int32Array.from(facts.returnTypes, () => 3) } + : facts; + let reason: string | undefined; + try { + linkIl2cppBinary(bytes, metadata, required); + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + reason = cause.reason; + } + expect(reason).toBe("metadata-registration-not-found"); + }, +); + +it.each([ + [ + // The real table is unloaded; a copy elsewhere is loaded instead. + "only a second long-enough run is loaded", + { metadataRegistration: false, adrpAdd: [[0xc20, 0xb00]] }, + (bytes: Buffer) => bytes.copy(bytes, 0xb00, 0x900, 0x920), + ], + [ + "a static MetadataRegistration matches its counts but not its type table", + { adrpAdd: [[0xc20, 0x900]] }, + (bytes: Buffer) => + bytes.writeBigUInt64LE(FIXTURE_BINARY_BASE + 0xdead0000n, 0x800 + 56), + ], + [ + "the loaded run reaches the end of its mapping", + { + metadataRegistration: false, + adrpAdd: [[0xc20, 0x900]], + splitSegmentAt: 0x920, + }, + () => undefined, + ], + [ + "a record's kind is the unassigned Il2CppTypeEnum value 0x17", + { metadataRegistration: false, adrpAdd: [[0xc20, 0x900]] }, + (bytes: Buffer) => bytes.writeUInt32LE(0x17 << 16, 0x980 + 16 + 8), + ], + [ + "a record's kind is the unassigned Il2CppTypeEnum value 0x1a", + { metadataRegistration: false, adrpAdd: [[0xc20, 0x900]] }, + (bytes: Buffer) => bytes.writeUInt32LE(0x1a << 16, 0x980 + 16 + 8), + ], + [ + // Three entries, one short of the four that type 3 needs. + "the loaded run is one entry short", + { metadataRegistration: false, adrpAdd: [[0xc20, 0x900]] }, + (bytes: Buffer) => bytes.writeBigUInt64LE(0n, 0x918), + ], + [ + "the ADRP+ADD pair uses register 31", + { metadataRegistration: false, adrpAdd: [[0xc20, 0x900]] }, + (bytes: Buffer) => { + bytes.writeUInt32LE((bytes.readUInt32LE(0xc20) | 0x1f) >>> 0, 0xc20); + bytes.writeUInt32LE((bytes.readUInt32LE(0xc24) | 0x3ff) >>> 0, 0xc24); + }, + ], +] as const)( + "keeps metadata-registration-not-found when %s, with the whole table referenced", + (_, options, patch) => { + const bytes = createIl2cppBinaryFixture(options); + patch(bytes); + let reason: string | undefined; + try { + linkWholeTable(bytes); + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + reason = cause.reason; + } + expect(reason).toBe("metadata-registration-not-found"); + }, +); + +it("does not choose a run that starts at its mapping start, where the table may continue backwards", () => { + // The table spans 0x900..0x920; the split puts its first entry in the + // previous mapping, and code loads 0x908, the first entry of the next. + const bytes = createIl2cppBinaryFixture({ + metadataRegistration: false, + splitSegmentAt: 0x908, + adrpAdd: [[0xc20, 0x908]], + }); + let reason: string | undefined; + try { + linkIl2cppBinary(bytes, metadata, { + ...facts, + returnTypes: Int32Array.from(facts.returnTypes, () => 1), + }); + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + reason = cause.reason; + } + expect(reason).toBe("metadata-registration-not-found"); +}); + +it("says why no MetadataRegistration or inferred type table was accepted", () => { + const message = (bytes: Buffer) => { + try { + linkWholeTable(bytes); + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + return cause.message; + } + throw new Error("expected a link failure"); + }; + const unusable = createIl2cppBinaryFixture({ adrpAdd: [[0xc20, 0x900]] }); + unusable.writeBigUInt64LE(FIXTURE_BINARY_BASE + 0xdead0000n, 0x800 + 56); + expect(message(unusable)).toContain( + "a MetadataRegistration counts 4 type definitions, but its type table", + ); + expect( + message(createIl2cppBinaryFixture({ metadataRegistration: false })), + ).toContain("no single code-loaded run of Il2CppType pointers"); +}); + +it("does not let a dyld-bound slot's placeholder decide the type-table run", () => { + const outcome = (placeholder: bigint) => { + // The last table slot (0x918) is bound by dyld; its stored word is only a + // placeholder. Remove the static registration and load the table by code. + const bytes = createIl2cppDyldBindFixture(placeholder, 0x918); + bytes.fill(0, 0x800, 0x880); + createIl2cppBinaryFixture({ adrpAdd: [[0xc20, 0x900]] }).copy( + bytes, + 0xc20, + 0xc20, + 0xc28, + ); + try { + return linkWholeTable(bytes).types_count; + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + return cause.reason; + } + }; + // A type-shaped placeholder and a zero one give the same refusal. + expect(outcome(FIXTURE_BINARY_BASE + 0x9b0n)).toBe( + "metadata-registration-not-found", + ); + expect(outcome(0n)).toBe("metadata-registration-not-found"); +}); + +it("does not let a dyld-bound record field decide the type-table run", () => { + const outcome = (bound: boolean) => { + // The int record (0x9b0) is the table's last entry. Binding the word + // that holds its kind leaves that kind to the loader, although the + // stored placeholder still reads as a valid kind. + const kindWord = createIl2cppBinaryFixture().readBigUInt64LE(0x9b8); + const bytes = bound + ? createIl2cppDyldBindFixture(kindWord, 0x9b8) + : createIl2cppBinaryFixture(); + bytes.fill(0, 0x800, 0x880); + createIl2cppBinaryFixture({ adrpAdd: [[0xc20, 0x900]] }).copy( + bytes, + 0xc20, + 0xc20, + 0xc28, + ); + try { + return linkWholeTable(bytes).types_count; + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + return cause.reason; + } + }; + expect(outcome(false)).toBe(4); + expect(outcome(true)).toBe("metadata-registration-not-found"); +}); + +it("keeps metadata-registration-not-found when x86-64 code loads an address inside the run", () => { + const image = createIl2cppBinaryFixture({ metadataRegistration: false }); + image.set([0x48, 0x8d, 0x0d], 0xc40); + image.writeInt32LE(0x908 - (0xc40 + 7), 0xc43); + let reason: string | undefined; + try { + linkWholeTable(createElfFixture(image, { machine: 62 })); + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + reason = cause.reason; + } + expect(reason).toBe("metadata-registration-not-found"); +}); + +it.each([ + ["a later page", { pages: 2, code: 0xc00, array: 0x1500 }], + ["an earlier page", { pages: 3, code: 0x2c00, array: 0x500 }], +] as const)( + "follows an arm64 ADRP page delta to a module array on %s", + (_, layout) => { + const bytes = createIl2cppBinaryFixture({ + codeRegistration: false, + pages: layout.pages, + adrpAdd: [[layout.code, layout.array]], + }); + if (layout.array !== 0x500) { + bytes.copy(bytes, layout.array, 0x500, 0x510); + bytes.fill(0, 0x500, 0x510); + } + const linked = link(bytes); + expect(linked.code_gen_modules_array).toBe( + FIXTURE_BINARY_BASE + BigInt(layout.array), + ); + expect(linked.method_addresses).toEqual(expectedAddresses); + }, +); + +it("follows a negative arm64 ADRP page delta to the type table", () => { + const linked = linkWholeTable( + createIl2cppBinaryFixture({ + metadataRegistration: false, + pages: 3, + adrpAdd: [[0x2c20, 0x900]], + }), + ); + expect(linked.types_field).toBeNull(); + expect(linked.types_count).toBe(4); +}); + +it.each([ + ["destination", 0], + ["source", 5], +] as const)( + "keeps code-registration-not-found when the ADD %s register differs from the ADRP register", + (_, shift) => { + const bytes = createIl2cppBinaryFixture({ + codeRegistration: false, + adrpAdd: [[0xc00, 0x500]], + }); + const add = bytes.readUInt32LE(0xc04); + // The fixture pair uses x8; write x9 into one register field of the ADD. + bytes.writeUInt32LE(((add & ~(0x1f << shift)) | (9 << shift)) >>> 0, 0xc04); + expect(failure(bytes).reason).toBe("code-registration-not-found"); + }, +); + +it("fails closed at the second type-table candidate when the metadata references few type indices", () => { + const bytes = createIl2cppBinaryFixture({ + metadataRegistration: false, + pages: 2, + adrpAdd: [[0xc20, 0x900]], + }); + // 65 isolated pointers to the int record, each a one-entry run. + for (let index = 0; index < 65; index++) + bytes.writeBigUInt64LE(FIXTURE_BINARY_BASE + 0x9b0n, 0x1000 + index * 16); + const outcome = (required: typeof facts) => { + try { + return linkIl2cppBinary(bytes, metadata, required).types_count; + } catch (cause) { + if (!(cause instanceof Il2cppBinaryFormatFailure)) throw cause; + return cause.reason; + } + }; + // Fixture methods reference only type 0, so every isolated pointer is a + // candidate run, and the second one ends the search. + expect(outcome(facts)).toBe("metadata-registration-not-found"); + // Referencing type 3 requires four entries; the isolated pointers drop out. + expect( + outcome({ + ...facts, + returnTypes: Int32Array.from(facts.returnTypes, () => 3), + }), + ).toBe(4); +}); + it("refuses to choose among too many candidate module arrays", () => { const base = createIl2cppBinaryFixture({ codeRegistration: false, @@ -1066,6 +1457,31 @@ it.each< expect(genericFailure(options, version)).toEqual({ reason, kind }); }); +it("refuses generic linking when the type table was inferred", () => { + // Without a static MetadataRegistration, genericMethodTable has no anchor. + const decoded = genericFacts(31); + expect(() => + linkIl2cppBinary( + createIl2cppBinaryFixture({ + metadataRegistration: false, + adrpAdd: [[0xc20, 0x900]], + }), + decoded.metadata, + { + ...decoded.link, + returnTypes: Int32Array.from(decoded.link.returnTypes, () => 3), + }, + undefined, + { genericInstances: true }, + ), + ).toThrow( + expect.objectContaining({ + reason: "generic-methods-unsupported", + message: expect.stringContaining("no static MetadataRegistration"), + }), + ); +}); + it("keeps contradicting rows unknown and reports the record address", () => { const issues = (options: GenericMethodFixtureOptions) => { const generic = linkGeneric(options).generic_methods!; diff --git a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts index 536b6751fc..3688136b39 100644 --- a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts +++ b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts @@ -1,15 +1,17 @@ -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; import { linkIl2cppBinary } from "../../../src/il2cpp/Il2cppBinaryFile.js"; import { isMapped, openImageSteps, readPointer, readSlot, + readUInt32, } from "../../../src/il2cpp/Il2cppBinaryImage.js"; import { readIl2cppMetadataForLinking } from "../../../src/il2cpp/Il2cppMetadataFile.js"; import { createElfFixture, createIl2cppBinaryFixture, + createIl2cppTypeScanFixture, FIXTURE_BINARY_BASE, } from "../../fixtures/il2cpp/binary.js"; import { createLinkedIl2cppMetadataFixture } from "../../fixtures/il2cpp/metadata.js"; @@ -61,6 +63,42 @@ const withLoadRanges = ( return result; }; +it("reuses admitted mappings for repeated file-backed loader reads", () => { + const original = createIl2cppBinaryFixture(); + const image = open(createElfFixture(original, { zeroFilledTable: true })); + const mappings = vi.spyOn(image.container.mappings, Symbol.iterator); + const zeroFills = vi.spyOn(image.container.zero_fills, "some"); + try { + for (let index = 0; index < 32; index++) { + expect(readSlot(image, FIXTURE_BINARY_BASE + 0x900n)).toBe( + original.readBigUInt64LE(0x900), + ); + expect(readUInt32(image, FIXTURE_BINARY_BASE + 0x988n)).toBe( + original.readUInt32LE(0x988), + ); + expect(isMapped(image, FIXTURE_BINARY_BASE + 0x980n, 12)).toBe(true); + } + expect(mappings).not.toHaveBeenCalled(); + expect(zeroFills).not.toHaveBeenCalled(); + } finally { + mappings.mockRestore(); + zeroFills.mockRestore(); + } +}); + +it("links one code-loaded type table among short runs across many mappings", () => { + const bytes = createIl2cppTypeScanFixture(64, 128); + const snapshot = Buffer.from(bytes); + const result = linkIl2cppBinary(bytes, metadata, { + ...facts, + returnTypes: Int32Array.from(facts.returnTypes, () => 3), + }); + expect(result.types_field).toBeNull(); + expect(result.types_count).toBe(4); + expect(result.return_types.every((row) => row.name === "int")).toBe(true); + expect(bytes).toEqual(snapshot); +}); + it.each([183, 62])( "links agreeing split ELF RELA spans without changing input bytes (machine %s)", (machine) => { diff --git a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts index 2f87c64627..c15d9c897b 100644 --- a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts @@ -643,6 +643,99 @@ it("retains both observed file identities and the selected architecture when nat } }); +it.each([ + { + architecture: "x86-64", + name: "libil2cpp.so", + binary: () => + createElfFixture( + createIl2cppBinaryFixture({ + codeRegistration: false, + codeReference: true, + }), + { machine: 62 }, + ), + named: "RIP-relative LEA/MOV", + absent: "ADRP+ADD", + }, + { + architecture: "arm64", + name: "GameAssembly.dylib", + binary: () => + createIl2cppBinaryFixture({ + codeRegistration: false, + adrpAdd: [[0xc00, 0x500]], + }), + named: "ADRP+ADD", + absent: "RIP-relative LEA/MOV", + }, +])( + "names the $architecture instruction form behind an inferred module array", + async ({ name, binary, named, absent }) => { + const root = await createTestTempDirectory("rea-il2cpp-inferred-form-"); + const path = join(root, "global-metadata.dat"), + binary_path = join(root, name); + await writeFile(path, createLinkedIl2cppMetadataFixture().bytes); + await writeFile(binary_path, binary()); + const provider = new Il2cppMetadataProvider(); + try { + const result = await provider.inspect({ path, binary_path }); + if (!result.ok) throw result.error; + const inferred = result.value.limitations.filter((line) => + line.includes("static CodeRegistration pair was not resolved"), + ); + expect(inferred).toHaveLength(1); + expect(inferred[0]).toContain(named); + expect(inferred[0]).not.toContain(absent); + } finally { + await provider.close(); + } + }, +); + +it("states that an inferred type table's count is a run length", async () => { + const root = await createTestTempDirectory("rea-il2cpp-inferred-types-"); + const path = join(root, "global-metadata.dat"), + binary_path = join(root, "GameAssembly.dylib"); + // A field typed as type 3 makes the metadata reference the whole + // four-entry table, as real builds reference most of theirs. + const fixture = createLinkedIl2cppMetadataFixture(); + const metadata = Buffer.from(fixture.bytes); + metadata.writeUInt8(3, fixture.section("fields").offset + 4); + await writeFile(path, metadata); + await writeFile( + binary_path, + createIl2cppBinaryFixture({ + metadataRegistration: false, + adrpAdd: [[0xc20, 0x900]], + }), + ); + const provider = new Il2cppMetadataProvider(); + try { + const result = await provider.inspect({ path, binary_path }); + if (!result.ok) throw result.error; + expect(result.value.binary).toMatchObject({ + types_field: null, + types_count: 4, + code_gen_modules_field: expect.any(String), + }); + const inferred = result.value.limitations.filter((line) => + line.includes( + "No static MetadataRegistration had the type-definition counts", + ), + ); + expect(inferred).toHaveLength(1); + expect(inferred[0]).toContain("ADRP+ADD"); + expect( + result.value.limitations.some((line) => + line.includes("static CodeRegistration pair was not resolved"), + ), + ).toBe(false); + } finally { + await provider.close(); + } +}); + it("reports generic method instances only when generic_instances is set", async () => { const root = await createTestTempDirectory("rea-il2cpp-provider-generic-"); const path = join(root, "global-metadata.dat"); diff --git a/tests/fixtures/il2cpp/binary.ts b/tests/fixtures/il2cpp/binary.ts index 384497377a..db9eecb78b 100644 --- a/tests/fixtures/il2cpp/binary.ts +++ b/tests/fixtures/il2cpp/binary.ts @@ -29,6 +29,10 @@ export interface Il2cppBinaryFixtureOptions { readonly duplicateCodeRegistration?: boolean; /** Write a RIP-relative `lea rax, [module array]`, as an inlined build loads it. */ readonly codeReference?: boolean; + /** Write `adrp x8, page; add x8, x8, #pageoff` at each file offset, loading the target offset. */ + readonly adrpAdd?: readonly (readonly [at: number, target: number])[]; + /** 4 KiB pages in the Mach-O image; extra pages are zero, past every table. */ + readonly pages?: number; /** Omit the MetadataRegistration type table. */ readonly metadataRegistration?: boolean; /** Write a second distinct MetadataRegistration candidate. */ @@ -131,20 +135,24 @@ const METHOD_SPECS = 0xb80; const GENERIC_METHOD_TABLE = 0xbc0; const GENERIC_METHOD_POINTERS = 0xd00; +const imageSize = (options: Il2cppBinaryFixtureOptions) => + SIZE * (options.pages ?? 1); + export const createIl2cppBinaryFixture = ( options: Il2cppBinaryFixtureOptions = {}, ): Buffer => { - const bytes = Buffer.alloc(SIZE); + const size = imageSize(options); + const bytes = Buffer.alloc(size); const address = (offset: number) => FIXTURE_BINARY_BASE + BigInt(offset); const chained = options.chainedFixups === true; const split = options.splitSegmentAt; // LC_SEGMENT_64 records, read-execute; a split gives two adjacent ones. const segments: readonly [name: string, start: number, end: number][] = split === undefined - ? [["__TEXT", 0, SIZE]] + ? [["__TEXT", 0, size]] : [ ["__TEXT", 0, split], - ["__DATA", split, SIZE], + ["__DATA", split, size], ]; const commands = segments.length + (chained ? 1 : 0); // Mach-O 64-bit header: arm64 dylib with its load commands. @@ -214,6 +222,21 @@ export const createIl2cppBinaryFixture = ( bytes.set([0x48, 0x8d, 0x05], at); bytes.writeInt32LE(MODULE_ARRAY - (at + 7), at + 3); } + for (const [at, target] of options.adrpAdd ?? []) { + const pc = address(at), + to = address(target); + const pages = Number(((to >> 12n) - (pc >> 12n)) & 0x1fffffn); + // ADRP x8: immlo in bits 29-30, immhi in bits 5-23. + bytes.writeUInt32LE( + (0x90000008 | ((pages & 3) << 29) | ((pages >> 2) << 5)) >>> 0, + at, + ); + // ADD x8, x8, #pageoff (64-bit, unshifted imm12). + bytes.writeUInt32LE( + (0x91000108 | (Number(to & 0xfffn) << 10)) >>> 0, + at + 4, + ); + } if (options.codeRegistration !== false) { bytes.writeBigUInt64LE(2n, REGISTRATION); bytes.writeBigUInt64LE(address(MODULE_ARRAY), REGISTRATION + 8); @@ -591,6 +614,43 @@ export const createElfFixture = ( return file; }; +/** A code-loaded type table plus isolated one-entry candidate runs in disjoint ELF mappings. */ +export const createIl2cppTypeScanFixture = ( + mappingCount: number, + bytesPerMapping: number, +): Buffer => { + const image = createIl2cppBinaryFixture({ + metadataRegistration: false, + adrpAdd: [[0xc20, 0x900]], + }); + const original = createElfFixture(image); + const candidates = Buffer.alloc(mappingCount * bytesPerMapping); + for (let at = 0; at < candidates.length; at += 16) + candidates.writeBigUInt64LE(FIXTURE_BINARY_BASE + 0x9b0n, at); + // The original dynamic segment and complete payload retain their addresses. + // Move only the program header table to the unmapped end of the file. + const headers: Buffer[] = [Buffer.from(original.subarray(120, 176))]; + for (let index = 0; index <= mappingCount; index++) { + const offset = + index === 0 ? 0 : original.length + (index - 1) * bytesPerMapping; + const size = index === 0 ? original.length : bytesPerMapping; + const header = Buffer.alloc(56); + header.writeUInt32LE(1); + header.writeUInt32LE(index === 0 ? 7 : 6, 4); + header.writeBigUInt64LE(BigInt(offset), 8); + header.writeBigUInt64LE(FIXTURE_BINARY_BASE + BigInt(offset), 16); + header.writeBigUInt64LE(FIXTURE_BINARY_BASE + BigInt(offset), 24); + header.writeBigUInt64LE(BigInt(size), 32); + header.writeBigUInt64LE(BigInt(size), 40); + header.writeBigUInt64LE(1n, 48); + headers.push(header); + } + const result = Buffer.concat([original, candidates, ...headers]); + result.writeBigUInt64LE(BigInt(original.length + candidates.length), 32); + result.writeUInt16LE(headers.length, 56); + return result; +}; + export interface GenericRegistrationImageOptions { /** Let the 104-byte layout bound the rows as well as the 120-byte one. */ readonly ambiguous?: boolean; From aced9fa9b4898744aa1af0c5f336382da7c9ca7e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:19:59 +0800 Subject: [PATCH 10/37] fix(wasm): retain acquired inputs across failed inspection Keep completed module and candidate identities, glue observations, and the admitted tool profile when WABT validation or later inspection fails. Recover stable reads from their existing owner when descriptor finalization fails; mark unparsed glue explicitly and preserve validation uncertainty. Forward partial Evidence through output/cleanup errors and preserve pre-abort reasons. Exercise producer failures, partial acquisition, decoding, finalization and cleanup retry with simulated owned WABT commands. Public CLI and stdio MCP coverage follows the retained Evidence reference through get_evidence_bundle. Real WABT 1.0.42 and the Windows shell-fixture lane were not verified. --- docs/wasm-artifacts.md | 13 ++ src/wasm/WabtArtifactProvider.test.ts | 172 +++++++++++++++ src/wasm/WabtArtifactProvider.ts | 200 ++++++++++++------ .../cli/wasmAcquiredInputsCli.test.ts | 163 ++++++++++++++ 4 files changed, 489 insertions(+), 59 deletions(-) create mode 100644 tests/boundary/cli/wasmAcquiredInputsCli.test.ts diff --git a/docs/wasm-artifacts.md b/docs/wasm-artifacts.md index ff58d373d4..8b7f9322eb 100644 --- a/docs/wasm-artifacts.md +++ b/docs/wasm-artifacts.md @@ -71,6 +71,19 @@ selected byte SHA-256 and size, section payload ranges, full header/detail outpu and its UTF-8 SHA-256. Producer format drift and invalid or truncated modules are failures, with validation diagnostics and partial Evidence retaining selected byte identity and tool provenance. +If inspection stops after selected files were read, partial Evidence also keeps +the completed candidate file identities and glue parse observations. A stable +read remains observed when descriptor finalization later fails; cleanup still +requires retry. Partial Evidence for a glue read whose finalization fails +before decoding retains its digest with `not-performed` parse status. Invalid +UTF-8 retains the digest with failed parse status. Neither case invents +references, and a failed file read contributes no invented facts. +These independent observations do not establish that the selected WASM module is +valid: partial results state whether WABT validation is unknown, rejected the +module, or succeeded before a later inspection failure. +CLI errors include this partial Evidence inline. MCP errors can return its +`retained-evidence` reference in `details.partial_observation`; use +`get_evidence_bundle` with `detail: "complete"` to retrieve the matching record. Core validation remains authoritative when an optional custom section cannot be interpreted. WABT warning output is preserved in `custom_section_warnings`; diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index ba584f68c7..5cbb8cf284 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -436,6 +436,29 @@ it("returns actionable unavailable without creating a workspace when tools are a }); }); +it("preserves a pre-aborted caller reason without acquiring WASM facts", async () => { + const reason = { source: "caller cancelled before provider admission" }; + const controller = new AbortController(); + controller.abort(reason); + const createRuntime = vi.fn(() => { + throw new Error("Pre-aborted inspection must not create a workspace"); + }); + const provider = new WabtArtifactProvider({}, undefined, createRuntime); + try { + const result = await provider.inspect( + { path: "/not-read.wasm", glue_paths: [], candidate_paths: [] }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(result.error.cause).toBe(reason); + expect(result.error.partialObservation).toBeUndefined(); + expect(createRuntime).not.toHaveBeenCalled(); + } finally { + await provider.close(); + } +}); + it.each([ { tool: "wasm-validate", stdout: "1.0.41\n", stderr: "" }, { tool: "wasm-objdump", stdout: "", stderr: "" }, @@ -559,6 +582,21 @@ it("retries a retained input descriptor before admitting the next WABT operation cleanupIncomplete: true, cleanupResources: [path], cause: { message: "injected WABT input close failure" }, + partialObservation: { + normalized_result: { + artifact: { + path, + sha256: createHash("sha256") + .update(Buffer.alloc(76, 1)) + .digest("hex"), + bytes: 76, + }, + candidates: [{ path }], + glue: [], + validation: "unknown", + tool_profile: { release: "1.0.42" }, + }, + }, }, }); expect(fileCloseFailure.attempts).toBe(1); @@ -979,6 +1017,140 @@ it.each(outcomeModes)( "owns private artifact bytes and retains %s outcome", (mode) => runOwnedArtifactOutcome(mode), ); + +it.each([ + { + stage: "validation rejection", + mode: "invalid", + validation: "rejected-by-selected-profile", + }, + { stage: "candidate admission", mode: "success", validation: "unknown" }, + { stage: "glue decoding", mode: "success", validation: "unknown" }, + { stage: "glue finalization", mode: "success", validation: "unknown" }, + { stage: "producer output", mode: "malformed-wat", validation: "valid" }, +] as const)( + "retains acquired file facts after $stage without claiming more module coverage", + async ({ stage, mode, validation }) => { + // Owned Node commands supply WABT-shaped responses; this checks provider + // Evidence retention, not real WABT acceptance of the fixture bytes. + const fixture = await createWabtArtifactFixture(); + const candidate = join(fixture.selectedRoot, "candidate.wasm"); + const candidateBytes = Buffer.from("independent candidate bytes"); + await writeFile(candidate, candidateBytes); + const missing = join(fixture.selectedRoot, "missing.wasm"); + const gluePath = join(fixture.selectedRoot, "glue.js"); + const glueSource = 'fetch("./selected.wasm");'; + const glueBytes = + stage === "glue decoding" ? Buffer.from([0xff]) : Buffer.from(glueSource); + await writeFile(gluePath, glueBytes); + const provider = new WabtArtifactProvider( + { REA_WABT_BIN_DIRECTORY: fixture.selectedRoot }, + (input) => + launchWabtOutcome(input, mode, { + ...fixture, + controller: new AbortController(), + }), + ); + if (stage === "glue finalization") { + fileCloseFailure.selectedPath = gluePath; + fileCloseFailure.attempts = 0; + fileCloseFailure.failFirst = true; + } + try { + const result = await provider.inspect({ + path: fixture.path, + candidate_paths: + stage === "candidate admission" ? [candidate, missing] : [candidate], + glue_paths: stage === "candidate admission" ? [] : [gluePath], + }); + if (result.ok) throw new Error("Expected the selected failure stage"); + expect(result.error._tag).toBe( + validation === "rejected-by-selected-profile" + ? "AnalysisInputError" + : validation === "unknown" + ? "AnalysisInputError" + : "AnalysisOutputError", + ); + expect(result.error.partialObservation).toMatchObject({ + normalized_result: { + artifact: { + path: fixture.path, + sha256: createHash("sha256").update(fixture.bytes).digest("hex"), + }, + validation, + tool_profile: { release: "1.0.42" }, + candidates: [ + { path: fixture.path }, + { + path: candidate, + sha256: createHash("sha256").update(candidateBytes).digest("hex"), + }, + ], + glue: + stage === "candidate admission" + ? [] + : [ + { + artifact: { + path: gluePath, + sha256: createHash("sha256") + .update(glueBytes) + .digest("hex"), + }, + parse_status: + stage === "glue finalization" + ? "not-performed" + : stage === "glue decoding" + ? "failed" + : "complete", + references: + stage === "glue decoding" || stage === "glue finalization" + ? [] + : [ + { + value: "./selected.wasm", + candidate_paths: [fixture.path], + }, + ], + }, + ], + }, + }); + if (validation === "rejected-by-selected-profile") + expect(result.error.partialObservation).toMatchObject({ + normalized_result: { + exit_code: 1, + stderr: "module.wasm: invalid bytes\n", + }, + }); + if (validation === "unknown") + expect(result.error).toMatchObject({ + issues: [ + { + path: + stage === "candidate admission" + ? ["candidate_paths", 1] + : ["glue_paths", 0], + }, + ], + }); + if (stage === "glue finalization") { + expect(result.error).toMatchObject({ + cleanupIncomplete: true, + cleanupResources: [gluePath], + }); + expect(fileCloseFailure.attempts).toBe(1); + await provider.close(); + expect(fileCloseFailure.attempts).toBe(2); + expect(result.error.cleanupIncomplete).toBe(false); + } + } finally { + await provider.close(); + fileCloseFailure.selectedPath = ""; + fileCloseFailure.failFirst = false; + } + }, +); it.each([null, { reason: "caller-owned object" }, undefined])( "preserves finalization abort reason %j through cleanup recovery", (reason) => runOwnedArtifactOutcome("cancelled-close-failure", reason), diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index 381c2f0679..79253f9475 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -8,7 +8,11 @@ import type { WasmArtifactExecution, WasmArtifactPort, } from "../application/wasm/WasmArtifactPort.js"; -import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { + completedStableArtifact, + readStableArtifact, + type StableArtifactSnapshot, +} from "../artifacts/readStableArtifact.js"; import { ArtifactResourceScope, ArtifactResourceRetryFailure, @@ -36,6 +40,7 @@ import { wasmArtifactSchema, DEFAULT_WASM_COMMAND_TIMEOUT_MS, type InspectWasmArtifactInput, + type WasmArtifact, } from "../domain/wasm/wasmArtifact.js"; import { err, ok, type Result } from "../domain/result.js"; import { runOwnedCommand } from "../process/OwnedCommand.js"; @@ -56,6 +61,13 @@ import { parseWabtWat } from "./WabtWat.js"; import { associateWasmGlue } from "./WasmGlueAdapter.js"; type Launcher = NonNullable[2]>["launcher"]; +type AcquiredGlue = + | WasmArtifact["glue"][number] + | { + readonly artifact: WasmArtifact["glue"][number]["artifact"]; + readonly parse_status: "not-performed"; + readonly references: []; + }; const OPERATION = "inspect_wasm_artifact"; const digest = (text: string) => createHash("sha256").update(text).digest("hex"); @@ -122,10 +134,71 @@ export class WabtArtifactProvider implements WasmArtifactPort { path: input.path, field: ["path"], }; + let toolProfile: WasmArtifact["tool_profile"] | undefined; + let artifact: WasmArtifact["artifact"] | undefined; + const candidates: WasmArtifact["candidates"] = []; + const glue: AcquiredGlue[] = []; + let validationState: "unknown" | "valid" = "unknown"; + const acquiredFacts = () => ({ + ...(artifact === undefined ? {} : { artifact }), + ...(toolProfile === undefined ? {} : { tool_profile: toolProfile }), + candidates: [...candidates], + glue: [...glue], + }); + const acquiredEvidence = () => + toolProfile === undefined + ? undefined + : createEvidence( + artifact === undefined + ? undefined + : { + path: artifact.path, + format: "file", + sha256: artifact.sha256, + }, + WABT_PROVIDER_IDENTITY, + { + operation: OPERATION, + parameters: input, + result: { ...acquiredFacts(), validation: validationState }, + confidence: "observed", + locations: [{ kind: "artifact-path", path: input.path }], + limitations: [ + artifact === undefined + ? "The selected WASM byte identity was not acquired. The admitted WABT tool profile does not establish module validity or runtime behavior." + : validationState === "valid" + ? "WABT validated the selected module, but inspection did not complete. Acquired file identities and glue references do not establish runtime loading or execution." + : "WABT validation has no completed result. Acquired file identities and glue references do not establish module validity, runtime loading or execution.", + ], + }, + ); + const readSelectedInput = async ( + path: string, + onCompleted: (snapshot: StableArtifactSnapshot) => void, + ): Promise => { + try { + const snapshot = await readStableArtifact( + path, + WABT_LIMITS.inputBytes, + options?.signal, + undefined, + this.#resources, + ); + onCompleted(snapshot); + return snapshot; + } catch (cause: unknown) { + const completed = completedStableArtifact(cause); + if (completed !== undefined) onCompleted(completed); + throw cause; + } + }; const commandTimeoutMs = input.command_timeout_ms ?? DEFAULT_WASM_COMMAND_TIMEOUT_MS; try { - if (options?.signal?.aborted) throw new AnalysisCancelledError(OPERATION); + if (options?.signal?.aborted) + throw new AnalysisCancelledError(OPERATION, { + cause: options.signal.reason, + }); const directory = this.environment.REA_WABT_BIN_DIRECTORY ?? ""; if (!isAbsolute(directory)) throw new Error( @@ -268,7 +341,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { versionProbe = undefined; } // Success and validation failure report the same admitted tool/execution facts. - const toolProfile = { + toolProfile = { source: "https://github.com/WebAssembly/wabt", release: WABT_VERSION, source_commit: "ff0ef7e0009402740c805a9744c09b05be063e48", @@ -277,66 +350,70 @@ export class WabtArtifactProvider implements WasmArtifactPort { output_budget_bytes: WABT_LIMITS.outputBytes, }; phase = "artifact"; - const selected = await readStableArtifact( - input.path, - WABT_LIMITS.inputBytes, - options?.signal, - undefined, - this.#resources, - ); - const artifact = { - path: input.path, - sha256: selected.sha256, - bytes: selected.bytes.length, - }; - const candidates = []; + const selected = await readSelectedInput(input.path, (snapshot) => { + const observed = { + path: input.path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }; + artifact = observed; + candidates.push(observed); + }); for (const path of new Set([input.path, ...input.candidate_paths])) { + if (path === input.path) continue; artifactInput = { path, - field: - path === input.path - ? ["path"] - : ["candidate_paths", input.candidate_paths.indexOf(path)], + field: ["candidate_paths", input.candidate_paths.indexOf(path)], }; - const bytes = - path === input.path - ? selected - : await readStableArtifact( - path, - WABT_LIMITS.inputBytes, - options?.signal, - undefined, - this.#resources, - ); - candidates.push({ + await readSelectedInput(path, (snapshot) => { + candidates.push({ + path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }); + }); + } + for (const [index, path] of input.glue_paths.entries()) { + artifactInput = { path, field: ["glue_paths", index] }; + const glueIndex = glue.length; + const bytes = await readSelectedInput(path, (snapshot) => { + glue.push({ + artifact: { + path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }, + parse_status: "not-performed", + references: [], + }); + }); + const glueArtifact = { path, sha256: bytes.sha256, bytes: bytes.bytes.length, - }); - } - const glue = []; - for (const [index, path] of input.glue_paths.entries()) { - artifactInput = { path, field: ["glue_paths", index] }; - const bytes = await readStableArtifact( - path, - WABT_LIMITS.inputBytes, - options?.signal, - undefined, - this.#resources, - ); - // Keep source characters intact so locations match the hashed artifact. - const source = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(bytes.bytes); - glue.push({ - artifact: { path, sha256: bytes.sha256, bytes: bytes.bytes.length }, - ...associateWasmGlue( - source, - path, - candidates.map((candidate) => candidate.path), - ), - }); + }; + try { + // Keep source characters intact so locations match the hashed artifact. + const source = new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true, + }).decode(bytes.bytes); + glue[glueIndex] = { + artifact: glueArtifact, + ...associateWasmGlue( + source, + path, + candidates.map((candidate) => candidate.path), + ), + }; + } catch (cause: unknown) { + glue[glueIndex] = { + artifact: glueArtifact, + parse_status: "failed", + references: [], + }; + throw cause; + } } phase = "producer"; await writeFile(snapshotPath, selected.bytes, { @@ -365,9 +442,8 @@ export class WabtArtifactProvider implements WasmArtifactPort { operation: OPERATION, parameters: input, result: { - artifact, + ...acquiredFacts(), validation: "rejected-by-selected-profile", - tool_profile: toolProfile, exit_code: validation.exitCode ?? null, stdout: validation.stdout.text, stderr: validation.stderr.text, @@ -394,6 +470,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { }, ], ); + validationState = "valid"; const customSectionWarnings: { tool: (typeof commands)[number]["tool"]; text: string; @@ -473,7 +550,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { }; const { sections, headers, details } = parseWabtOutput( dump.stdout.text, - artifact.bytes, + selected.bytes.length, customSectionNames, debugModuleNames, ); @@ -530,6 +607,10 @@ export class WabtArtifactProvider implements WasmArtifactPort { ...(versionProbe === undefined ? {} : { versionProbe }), }, ); + if (failure.partialObservation === undefined) { + const partial = acquiredEvidence(); + if (partial !== undefined) failure.retainPartialObservation(partial); + } runtime?.retainCommandFailure(cause, failure, this.#resources); result = err( failure instanceof AnalysisOutputError && @@ -540,6 +621,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { capturedOutput, cleanup: failure.cleanup, diagnostics: failure.diagnostics, + partialObservation: failure.partialObservation, }).bindCleanupDependencies([failure]) : failure, ); diff --git a/tests/boundary/cli/wasmAcquiredInputsCli.test.ts b/tests/boundary/cli/wasmAcquiredInputsCli.test.ts new file mode 100644 index 0000000000..66a59c3856 --- /dev/null +++ b/tests/boundary/cli/wasmAcquiredInputsCli.test.ts @@ -0,0 +1,163 @@ +import { createHash } from "node:crypto"; +import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { analysisErrorProjectionSchema } from "../../../src/contracts/errorSchemas.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const sha256 = (bytes: Buffer | string): string => + createHash("sha256").update(bytes).digest("hex"); + +// These scripts exercise the public adapters with WABT-shaped responses. They +// do not establish how the real WABT release classifies the selected bytes. +cliTest.runIf(process.platform !== "win32")( + "retains selected, candidate, and glue facts after simulated WABT rejection through CLI and stdio MCP", + async ({ cli }) => { + const directory = await createTestTempDirectory("rea-wasm-acquired-"); + const tools = join(directory, "tools"); + await mkdir(tools); + for (const name of ["wasm-validate", "wasm-objdump", "wasm2wat"]) { + const path = join(tools, name); + const body = + name === "wasm-validate" + ? "echo 'fixture validation rejected' >&2\nexit 1" + : "exit 0"; + await writeFile( + path, + `#!/bin/sh +if [ "$1" = "--version" ]; then echo '1.0.42'; exit 0; fi +${body} +`, + { mode: 0o700 }, + ); + await chmod(path, 0o700); + } + const selected = join(directory, "selected.wasm"); + const candidate = join(directory, "candidate.wasm"); + const glue = join(directory, "glue.js"); + const selectedBytes = Buffer.from("0061736d01000000", "hex"); + const candidateBytes = Buffer.from("independent candidate bytes"); + const glueSource = 'fetch("./selected.wasm");'; + const environment = { + HOME: directory, + XDG_CONFIG_HOME: directory, + XDG_CACHE_HOME: directory, + REA_WABT_BIN_DIRECTORY: tools, + }; + await Promise.all([ + writeFile(selected, selectedBytes), + writeFile(candidate, candidateBytes), + writeFile(glue, glueSource), + ]); + const expected = { + artifact: { + path: selected, + sha256: sha256(selectedBytes), + bytes: selectedBytes.length, + }, + candidates: [ + { + path: selected, + sha256: sha256(selectedBytes), + bytes: selectedBytes.length, + }, + { + path: candidate, + sha256: sha256(candidateBytes), + bytes: candidateBytes.length, + }, + ], + glue: [ + { + artifact: { + path: glue, + sha256: sha256(glueSource), + bytes: Buffer.byteLength(glueSource), + }, + parse_status: "complete", + references: [ + { + value: "./selected.wasm", + candidate_paths: [selected], + }, + ], + }, + ], + validation: "rejected-by-selected-profile", + tool_profile: { release: "1.0.42" }, + exit_code: 1, + stderr: "fixture validation rejected\n", + }; + const invocation = await cli.run({ + arguments: [ + "inspect-wasm-artifact", + selected, + "--candidate_paths", + candidate, + "--glue_paths", + glue, + "--format", + "json", + ], + environment, + }); + expect(invocation.exitCode, invocation.stderr).toBe(1); + const cliError = analysisErrorProjectionSchema.parse(invocation.json); + expect(cliError).toMatchObject({ + code: "invalid_request", + details: { + partial_observation: { + normalized_result: expected, + }, + }, + }); + + const { client } = await connectLocalToolsMcp(environment); + const response = await client.callTool({ + name: "inspect_wasm_artifact", + arguments: { + path: selected, + candidate_paths: [candidate], + glue_paths: [glue], + }, + }); + const mcpError = parseMcpToolError(response).error; + expect(mcpError.code).toBe("invalid_request"); + const partialReference = z + .object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }) + .parse( + z + .object({ partial_observation: z.unknown() }) + .passthrough() + .parse(mcpError.details).partial_observation, + ); + const retainedReply = await client.callTool({ + name: "get_evidence_bundle", + arguments: { detail: "complete" }, + }); + expect(retainedReply.isError).not.toBe(true); + const retainedBundle = toolContract( + "get_evidence_bundle", + ).outputSchema.parse(retainedReply.structuredContent).result; + if ("kind" in retainedBundle) + throw new Error("Expected complete retained Evidence"); + const retained = parseEvidence( + retainedBundle.records.find( + (record) => record.evidence_id === partialReference.evidence_id, + ), + ); + expect(retained.evidence_id).toBe(partialReference.evidence_id); + expect(retained.normalized_result).toMatchObject(expected); + await client.ping(); + }, +); From 5ea2c280b8f21d8ca11c07bf7c987d1b2d8ba8aa Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:36:04 +0800 Subject: [PATCH 11/37] refactor(doctor): retain parsed configuration diagnostics Return native admission issues and an explicit Hopper timeout result from the configuration owner. Doctor consumes those parsed facts instead of raw timeout text plus a separate issue list. Reuse a successful aggregate parse; validate the individual timeout independently when other settings reject it. Preserve aggregate error precedence, JEB isolation, omission, constraints, causes and provider scope. Keep partial diagnostics when unrelated NUL, loader arguments, language selection or timeout settings are invalid. Validation: configuration and Doctor workflow regressions, mixed-invalid settings, full static/docs checks. No installed provider was exercised. --- docs/design/limits-and-recovery.md | 6 +- src/application/Doctor.test.ts | 98 ++++++++++++++++++++++------ src/application/Doctor.ts | 19 ++---- src/application/DoctorDiagnostics.ts | 17 ++--- src/config.test.ts | 39 +++++++++-- src/config/parseConfig.ts | 51 ++++++++++++--- 6 files changed, 172 insertions(+), 58 deletions(-) diff --git a/docs/design/limits-and-recovery.md b/docs/design/limits-and-recovery.md index 4ed7a49ace..94c507e6b2 100644 --- a/docs/design/limits-and-recovery.md +++ b/docs/design/limits-and-recovery.md @@ -18,8 +18,10 @@ default, any floor, the maximum, the schema, the parser, and the recovery text. Register that schema on `environmentSchema` in [`environment.ts`](https://github.com/morluto/rea/blob/main/src/config/environment.ts). `parseConfig` calls `parseEnvironment`. Doctor uses that same parser through -`nativeAnalysisConfigurationError`, so startup and `rea doctor` accept the -same values. Add the name to `SELECTED_REGISTRATION_SETTINGS` so `rea setup` +`inspectNativeAnalysisConfiguration`, which also preserves a supplied Hopper +timeout's parsed diagnostic when another setting is invalid. Startup and +`rea doctor` accept the same values. Add the name to +`SELECTED_REGISTRATION_SETTINGS` so `rea setup` copies it into the client registration. Unset keeps today's behavior. An invalid value is a `ConfigurationError` and projects as `configuration_invalid`. Do not fall back to the default. diff --git a/src/application/Doctor.test.ts b/src/application/Doctor.test.ts index 683634ed08..5a1b36cb71 100644 --- a/src/application/Doctor.test.ts +++ b/src/application/Doctor.test.ts @@ -12,6 +12,16 @@ import { HopperReleaseError } from "./HopperReleaseFeed.js"; import { MAC_HOPPER_SYSTEM_LAUNCHER } from "./MacHopperPaths.js"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +const configurationHost = ( + environment: Readonly, +): DoctorHost => ({ + ...systemDoctorHost({ environment: { ...environment } }), + ...host({ + installationPaths: () => Promise.resolve([]), + clientRegistrations: () => Promise.resolve([]), + }), +}); + describe("doctor Hopper startup timeout", () => { const timeoutHost = ( value: string | undefined, @@ -157,16 +167,6 @@ describe("doctor Hopper startup timeout", () => { }); describe("doctor native analysis configuration", () => { - const configurationHost = ( - environment: Readonly, - ): DoctorHost => ({ - ...systemDoctorHost({ environment: { ...environment } }), - ...host({ - installationPaths: () => Promise.resolve([]), - clientRegistrations: () => Promise.resolve([]), - }), - }); - it.each([ ["REA_ANALYSIS_PROVIDER", "Hopper"], ["REA_REGEX_MATCHING_DEADLINE_MS", "0"], @@ -202,9 +202,78 @@ describe("doctor native analysis configuration", () => { details: { setting }, }), ); + expect( + result.checks.some(({ name }) => name === "hopper-startup-timeout"), + ).toBe(false); }, ); + it.each([ + ["REA_GHIDRA_STARTUP_TIMEOUT_MS", "abc"], + ["HOPPER_LOADER_ARGS_JSON", "{"], + ["REA_GHIDRA_COMPILER_SPEC_ID", "gcc"], + ["CALLER_SETTING", "private\0value"], + ] as const)( + "keeps a valid supplied Hopper timeout when %s rejects native admission", + async (setting, value) => { + const result = await runDoctor( + undefined, + configurationHost({ + [setting]: value, + REA_HOPPER_STARTUP_TIMEOUT_MS: "120001", + }), + { providers: ["hopper"] }, + ); + expect(result.healthy).toBe(false); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: "hopper-startup-timeout", + ok: true, + detail: "REA_HOPPER_STARTUP_TIMEOUT_MS=120001", + }), + ); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: `runtime-configuration:${setting}`, + ok: false, + }), + ); + }, + ); + + it("reports an invalid Hopper timeout alongside an unrelated NUL admission refusal", async () => { + const result = await runDoctor( + undefined, + configurationHost({ + CALLER_SETTING: "private\0value", + REA_HOPPER_STARTUP_TIMEOUT_MS: "0", + }), + { providers: ["hopper"] }, + ); + expect(result.healthy).toBe(false); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: "hopper-startup-timeout", + ok: false, + detail: + "REA_HOPPER_STARTUP_TIMEOUT_MS: must be a positive decimal integer", + }), + ); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: "runtime-configuration:CALLER_SETTING", + ok: false, + detail: expect.stringContaining("NUL"), + }), + ); + expect( + result.scope_checks.filter( + ({ name }) => name === "hopper-startup-timeout", + ), + ).toHaveLength(1); + expect(JSON.stringify(result.checks)).not.toContain("private"); + }); + it("keeps a lowercase provider preference ready for the selected engine", async () => { const result = await runDoctor( undefined, @@ -257,15 +326,6 @@ describe("doctor native analysis configuration", () => { }); describe("doctor MCP startup readiness", () => { - const configurationHost = ( - environment: Readonly, - ): DoctorHost => ({ - ...systemDoctorHost({ environment: { ...environment } }), - ...host({ - installationPaths: () => Promise.resolve([]), - clientRegistrations: () => Promise.resolve([]), - }), - }); const alignedCodex: ClientRegistrationStatus = { client: "codex", config_path: "/home/user/.codex/config.toml", diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 8249899f41..2f55203220 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -11,9 +11,11 @@ import { constants } from "node:fs"; import { access, stat } from "node:fs/promises"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import type { RetainedResourceScope } from "./AnalysisProvider.js"; -import { nativeAnalysisConfigurationError } from "../config/parseConfig.js"; +import { + inspectNativeAnalysisConfiguration, + type NativeAnalysisConfigurationInspection, +} from "../config/parseConfig.js"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; -import type { ConfigurationSettingIssue } from "../domain/configurationErrors.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { join } from "node:path"; @@ -108,12 +110,11 @@ export interface DoctorHost { readonly runtime: JavaScriptRuntime; readonly homeDirectory: string; readonly configuredHopperPath?: string; - readonly configuredHopperStartupTimeout?: string; /** * Settings `parseConfig` rejects before native analysis starts. * Undefined means this host did not run that admission. */ - readonly configurationIssues?: readonly ConfigurationSettingIssue[]; + readonly nativeConfiguration?: NativeAnalysisConfigurationInspection; readonly configuredIlspyCmdPath?: string; macosVersion(): Promise; linuxDistribution(): Promise; @@ -539,7 +540,7 @@ export const systemDoctorHost = ( const platform = options.platform ?? process.platform; const architecture = options.architecture ?? process.arch; const environment = options.environment ?? process.env; - const configurationError = nativeAnalysisConfigurationError( + const nativeConfiguration = inspectNativeAnalysisConfiguration( snapshotEnvironment(environment, platform), ); const hostExecFileOutput = options.execFileOutput ?? execFileOutput; @@ -576,13 +577,7 @@ export const systemDoctorHost = ( ...(environment.HOPPER_LAUNCHER_PATH === undefined ? {} : { configuredHopperPath: environment.HOPPER_LAUNCHER_PATH }), - ...(environment.REA_HOPPER_STARTUP_TIMEOUT_MS === undefined - ? {} - : { - configuredHopperStartupTimeout: - environment.REA_HOPPER_STARTUP_TIMEOUT_MS, - }), - configurationIssues: configurationError?.settings ?? [], + nativeConfiguration, ...(environment.REA_ILSPY_CMD_PATH === undefined ? {} : { configuredIlspyCmdPath: environment.REA_ILSPY_CMD_PATH }), diff --git a/src/application/DoctorDiagnostics.ts b/src/application/DoctorDiagnostics.ts index 3cbfc7e784..0713c6b602 100644 --- a/src/application/DoctorDiagnostics.ts +++ b/src/application/DoctorDiagnostics.ts @@ -2,7 +2,6 @@ import { SUPPORTED_NODE_VERSION_PROSE, supportsJavaScriptRuntime, } from "../domain/runtimeVersion.js"; -import { providerStartupTimeoutSchema } from "../config/providerStartupTimeout.js"; import { hopperLauncherDiscovery } from "./HopperLauncherDiscovery.js"; import { LINUX_HOPPER_SETUP_HOSTS, supportsReaHost } from "./LinuxHopper.js"; import type { @@ -193,16 +192,14 @@ const hopperCheck = ( const optionalHopperStartupTimeoutCheck = ( host: DoctorHost, ): DoctorCheck | undefined => { - if (host.configuredHopperStartupTimeout === undefined) return undefined; - const parsed = providerStartupTimeoutSchema.safeParse( - host.configuredHopperStartupTimeout, - ); + const parsed = host.nativeConfiguration?.configuredHopperStartupTimeout; + if (parsed === undefined) return undefined; return check( "hopper-startup-timeout", - parsed.success, - parsed.success - ? `REA_HOPPER_STARTUP_TIMEOUT_MS=${String(parsed.data)}` - : `REA_HOPPER_STARTUP_TIMEOUT_MS: ${parsed.error.issues.map(({ message }) => message).join("; ")}`, + parsed.ok, + parsed.ok + ? `REA_HOPPER_STARTUP_TIMEOUT_MS=${String(parsed.value)}` + : `REA_HOPPER_STARTUP_TIMEOUT_MS: ${parsed.error.settings.map(({ constraint }) => constraint).join("; ")}`, { classification: "config_drift", remediation: @@ -219,7 +216,7 @@ const optionalHopperStartupTimeoutCheck = ( const runtimeConfigurationChecks = ( host: DoctorHost, ): readonly DoctorCheck[] => { - const issues = host.configurationIssues; + const issues = host.nativeConfiguration?.issues; if (issues === undefined || issues.length === 0) return []; const grouped: { setting: string; constraints: string[] }[] = []; for (const issue of issues) { diff --git a/src/config.test.ts b/src/config.test.ts index a4261a5faf..939fcc4e88 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { defaultHopperLauncherPath, - nativeAnalysisConfigurationError, + inspectNativeAnalysisConfiguration, parseConfig, } from "./config/parseConfig.js"; import { @@ -100,16 +100,16 @@ describe("runtime configuration", () => { }); it("shares native admission with doctor and ignores the JEB-only timeout", () => { - expect(nativeAnalysisConfigurationError({})).toBeUndefined(); + expect(inspectNativeAnalysisConfiguration({})).toEqual({ issues: [] }); expect( - nativeAnalysisConfigurationError({ REA_JEB_MCP_TIMEOUT_MS: "abc" }), - ).toBeUndefined(); + inspectNativeAnalysisConfiguration({ REA_JEB_MCP_TIMEOUT_MS: "abc" }), + ).toEqual({ issues: [] }); expect(parseConfig({ REA_JEB_MCP_TIMEOUT_MS: "abc" }).ok).toBe(true); - const rejected = nativeAnalysisConfigurationError({ + const rejected = inspectNativeAnalysisConfiguration({ REA_ANALYSIS_PROVIDER: "Hopper", REA_GHIDRA_STARTUP_TIMEOUT_MS: "abc", }); - expect(rejected?.settings.map(({ setting }) => setting)).toEqual([ + expect(rejected.issues.map(({ setting }) => setting)).toEqual([ "REA_ANALYSIS_PROVIDER", "REA_GHIDRA_STARTUP_TIMEOUT_MS", ]); @@ -121,6 +121,33 @@ describe("runtime configuration", () => { ).toBe(false); }); + it("preserves the timeout's schema cause when aggregate admission stops at NUL", () => { + const inspected = inspectNativeAnalysisConfiguration({ + CALLER_SETTING: "private\0value", + REA_HOPPER_STARTUP_TIMEOUT_MS: "0", + }); + expect(inspected.issues).toEqual([ + { + setting: "CALLER_SETTING", + constraint: + "Environment variable names and values must not contain NUL bytes", + }, + ]); + const timeout = inspected.configuredHopperStartupTimeout; + if (timeout === undefined || timeout.ok) + throw new Error("Expected the configured timeout's independent refusal"); + expect(timeout.error.cause).toMatchObject({ + name: "ZodError", + issues: [{ message: "must be a positive decimal integer" }], + }); + expect(timeout.error.settings).toEqual([ + { + setting: "REA_HOPPER_STARTUP_TIMEOUT_MS", + constraint: "must be a positive decimal integer", + }, + ]); + }); + it("parses absolute BYO Ghidra and optional Java paths", () => { expect( parseConfig({ diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index 60235532b7..cdc35f9589 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -1,8 +1,12 @@ import { accessSync, constants, statSync } from "node:fs"; -import type { ConfigurationError } from "../domain/configurationErrors.js"; -import { ok, type Result } from "../domain/result.js"; +import { + ConfigurationError, + type ConfigurationSettingIssue, +} from "../domain/configurationErrors.js"; +import { err, ok, type Result } from "../domain/result.js"; import { parseEnvironment } from "./environment.js"; +import { providerStartupTimeoutSchema } from "./providerStartupTimeout.js"; import { homeDirectoryFromEnvironment } from "./homeDirectory.js"; import { DEFAULT_BRIDGE_MAX_REQUEST } from "./bridgeMaxRequestBytes.js"; import { DEFAULT_REGEX_MATCHING_DEADLINE } from "./regexMatchingDeadline.js"; @@ -48,15 +52,44 @@ const nativeAnalysisEnvironment = ( return appEnvironment; }; -/** - * The configuration error native commands return, without probing launchers. - * Doctor uses this so readiness and analysis admit the same settings. - */ -export const nativeAnalysisConfigurationError = ( +/** Native admission issues and an explicitly configured timeout's independent facts. */ +export interface NativeAnalysisConfigurationInspection { + readonly issues: readonly ConfigurationSettingIssue[]; + readonly configuredHopperStartupTimeout?: Result; +} + +/** Inspect native admission without probing launchers or losing mixed-invalid diagnostics. */ +export const inspectNativeAnalysisConfiguration = ( environment: Readonly>, -): ConfigurationError | undefined => { +): NativeAnalysisConfigurationInspection => { const parsed = parseEnvironment(nativeAnalysisEnvironment(environment)); - return parsed.ok ? undefined : parsed.error; + const issues = parsed.ok ? [] : parsed.error.settings; + const configuredTimeout = environment.REA_HOPPER_STARTUP_TIMEOUT_MS; + if (configuredTimeout === undefined) return { issues }; + if (parsed.ok) + return { + issues, + configuredHopperStartupTimeout: ok( + parsed.value.REA_HOPPER_STARTUP_TIMEOUT_MS, + ), + }; + // Another setting may reject the aggregate before this supplied timeout's + // facts are available. Its dedicated diagnostic still uses the same schema. + const timeout = providerStartupTimeoutSchema.safeParse(configuredTimeout); + return { + issues, + configuredHopperStartupTimeout: timeout.success + ? ok(timeout.data) + : err( + new ConfigurationError("Invalid REA environment configuration", { + cause: timeout.error, + settings: timeout.error.issues.map(({ message }) => ({ + setting: "REA_HOPPER_STARTUP_TIMEOUT_MS", + constraint: message, + })), + }), + ), + }; }; /** Parse provider configuration and runtime prerequisites at the composition root. */ From 835904e4ee44d380e6d52b08e32ecceeecbfbe03 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:36:05 +0800 Subject: [PATCH 12/37] refactor(evidence): carry producer-owned source records Carry the final artifact inspection's exact nested source Evidence through AnalysisExecution. Derive links in the shared Evidence builder instead of reparsing the result in CLI and MCP adapters. Retain validation for unknown and historical Evidence inputs. Move the shared source recorder into application ownership and migrate its consumers. Record sources before parents; keep source recording failures from publishing summaries. Decline snapshot entries whose source closure the current entry format cannot represent, while retaining investigation Evidence. Validation: actual ASAR regression failed before the change with missing links and passes afterward; exact nested identity, source recorder refusal, final-close cancellation and cleanup failure, historical authentication, snapshot controls and public CLI/MCP workflows. Fresh runtime/catalog build, full static/docs, discovery and duplication checks pass. Installed native providers and platform-specific skipped cases remain unverified. --- src/application/AnalysisProvider.ts | 5 + src/application/DirectAnalysis.ts | 37 ++----- .../binary/AnalysisSnapshotCache.test.ts | 57 +++++++++++ .../binary/AnalysisSnapshotCache.ts | 3 + src/application/binary/BinarySession.ts | 33 +++---- .../investigation/recordEvidenceSources.ts} | 10 +- src/artifacts/ArtifactProvider.ts | 6 +- src/domain/analysisObservation.ts | 4 + src/domain/artifactInspection.ts | 25 +++-- .../compareExportShapes.ts | 4 +- .../compareSourceToBundle.ts | 4 +- .../compareVersions.ts | 4 +- .../obligationLedger.ts | 4 +- .../projectMobileApplicationGraph.ts | 4 +- .../registerApplicationTools/traceFeature.ts | 4 +- .../traceSemantics.ts | 9 +- src/server/registerArtifactComparisonTool.ts | 4 +- src/server/registerArtifactTools.ts | 7 -- src/server/registerBundleComparisonTool.ts | 4 +- src/server/registerEvidenceTools.ts | 48 +++------ src/server/registerFunctionComparisonTool.ts | 4 +- .../compareManagedMembers.ts | 4 +- .../projectManagedApplicationGraph.ts | 4 +- .../verifyManagedNativeBoundaries.ts | 4 +- .../registerNativeFunctionMatchingTool.ts | 4 +- src/server/registerProcessComparisonTool.ts | 4 +- src/server/registerWasmTools.ts | 4 +- tests/boundary/mcp/artifactMcp.test.ts | 90 ++++++++++++++++- .../artifactProvider.directAnalysis.test.ts | 42 ++++++++ .../composition/directAnalysisCleanup.test.ts | 97 ++++++++++++++++++- 30 files changed, 394 insertions(+), 139 deletions(-) rename src/{server/sessionEvidence.ts => application/investigation/recordEvidenceSources.ts} (55%) diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index a0621d2afb..b216afa1f8 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -11,6 +11,7 @@ import type { AnalysisProfileCommitment } from "../domain/analysisProfile.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { Result } from "../domain/result.js"; import type { + Evidence, EvidenceLocation, EvidenceSubjectTarget, } from "../domain/evidence.js"; @@ -72,6 +73,7 @@ interface CreateAnalysisExecutionOptions { readonly relationshipCoverage?: RelationshipCoverage; readonly referencesTruncated?: boolean; readonly locations?: readonly EvidenceLocation[]; + readonly sourceEvidence?: readonly Evidence[]; readonly subject?: EvidenceSubjectTarget | null; } @@ -122,6 +124,9 @@ export function createAnalysisExecution( ? {} : { referencesTruncated: options.referencesTruncated }), locations: [...(options.locations ?? [])], + ...(options.sourceEvidence === undefined + ? {} + : { sourceEvidence: Object.freeze([...options.sourceEvidence]) }), ...(options.subject === undefined ? {} : { subject: options.subject }), }; } diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index acac584bd8..462d0a7100 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -8,10 +8,7 @@ import type { SessionProviderRoute } from "./binary/SessionProviderRouter.js"; import type { ResolvedSessionOpen } from "./binary/BinarySessionOpen.js"; import { silentLogger } from "../logger.js"; import type { Logger } from "pino"; -import { - createEvidence, - withEvidenceCallerInputPath, -} from "../domain/evidence.js"; +import { withEvidenceCallerInputPath } from "../domain/evidence.js"; import { createImmutableEvidenceBundle } from "../domain/evidenceBundle.js"; import type { Evidence } from "../domain/evidence.js"; import type { NativeToolName } from "../contracts/native/nativeToolContracts.js"; @@ -58,7 +55,6 @@ import { recordWorkflowUnknowns, } from "./WorkflowEvidence.js"; import type { AnalysisProviderSelector } from "../contracts/providerSelection.js"; -import { artifactInspectionResultSchema } from "../domain/artifactInspection.js"; import { resolveXrefsAddress } from "./XrefsAddressResolution.js"; import { AnalysisCancelledError, @@ -644,30 +640,17 @@ const executeAnalysisTool = async (input: { ); } if (!result.ok) return result; - const evidence = createEvidence( - result.value.subject === undefined - ? input.openedTarget - : (result.value.subject ?? undefined), - result.value.provider, + const evidence = createAnalysisExecutionEvidence( + tool, + input.evidenceParameters, { - operation: tool, - parameters: input.evidenceParameters, - result: result.value.result, - ...(result.value.analysisProfile === undefined - ? {} - : { analysisProfile: result.value.analysisProfile }), - rawResult: result.value.rawResult, - limitations: result.value.limitations, - ...(result.value.relationshipCoverage === undefined - ? {} - : { relationshipCoverage: result.value.relationshipCoverage }), - locations: result.value.locations, - evidenceLinks: - tool === "inspect_artifact" - ? artifactInspectionResultSchema.parse(result.value.result) - .evidence_links - : [], + ...result.value, + subject: + result.value.subject === undefined + ? input.openedTarget + : result.value.subject, }, + "observed", ); return ok({ evidence }); }; diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index f895fc9a15..39441cf936 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -930,6 +930,63 @@ it("does not replay pre-envelope call graphs under the revised workflow profile" ).toBe(false); }); +it("declines source-bearing snapshot entries while replaying omitted and empty source controls", () => { + const cache = new AnalysisSnapshotCache(); + const target = ANALYSIS_SNAPSHOT_TARGET; + const profile = ANALYSIS_SNAPSHOT_PROFILE; + const source = createEvidence(target, ANALYSIS_SNAPSHOT_PROVIDER, { + operation: "source", + parameters: {}, + result: "observed source", + }); + const parameters = { address: "0x1000", document: "fixture" }; + cache.record({ + target, + profile, + operation: "procedure_info", + parameters, + execution: createAnalysisExecution( + "parent observation", + ANALYSIS_SNAPSHOT_PROVIDER, + { + subject: target, + analysisProfile: profile, + sourceEvidence: [source], + }, + ), + }); + expect( + cache.lookup(target, profile, "procedure_info", parameters), + ).toBeUndefined(); + expect(cache.entries()).toEqual([]); + + for (const sourceEvidence of [undefined, []] as const) { + const control = { + address: sourceEvidence === undefined ? "0x2000" : "0x3000", + document: "fixture", + }; + cache.record({ + target, + profile, + operation: "procedure_info", + parameters: control, + execution: createAnalysisExecution( + "source-free observation", + ANALYSIS_SNAPSHOT_PROVIDER, + { + subject: target, + analysisProfile: profile, + ...(sourceEvidence === undefined ? {} : { sourceEvidence }), + }, + ), + }); + expect( + cache.lookup(target, profile, "procedure_info", control)?.result, + ).toBe("source-free observation"); + } + expect(cache.entries()).toHaveLength(2); +}); + describe("executions that report truncated references", () => { const record = (referencesTruncated: boolean | undefined) => { const cache = new AnalysisSnapshotCache(); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index dad37c58b3..4db71038ff 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -436,6 +436,9 @@ export class AnalysisSnapshotCache { // The snapshot format does not carry reference truncation; a replay would // present a capped read as complete, so it is not recorded. if (execution.referencesTruncated === true) return; + // Provider snapshot entries do not carry execution source records. Replay + // must not discard the source closure that supplies the parent's Evidence links. + if ((execution.sourceEvidence?.length ?? 0) > 0) return; const queryId = analysisQueryId( snapshotTarget(target), snapshotBinding(profile), diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index b9ac301fd2..908ecc82ec 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -13,7 +13,7 @@ import { import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../../domain/result.js"; import type { JsonValue } from "../../domain/jsonValue.js"; -import { createEvidence } from "../../domain/evidence.js"; +import { createAnalysisExecutionEvidence } from "../../domain/analysisObservation.js"; import type { AnalysisClient, AnalysisExecution, @@ -33,6 +33,7 @@ import { type SessionProviderRoute, } from "./SessionProviderRouter.js"; import { InvestigationRecords } from "../investigation/InvestigationRecords.js"; +import { recordEvidenceSources } from "../investigation/recordEvidenceSources.js"; import { BinarySessionRecords } from "./BinarySessionRecords.js"; import { binarySessionStatus } from "./BinarySessionStatus.js"; import { @@ -1256,26 +1257,24 @@ export class BinarySession name !== "inspect_keyed_archive" && name !== "trace_dylib_resolution" ) { - const evidence = createEvidence( - execution.value.subject === undefined - ? active.target - : (execution.value.subject ?? undefined), - execution.value.provider, + const evidence = createAnalysisExecutionEvidence( + name, + arguments_, { - operation: name, - parameters: arguments_, - result: execution.value.result, + ...execution.value, analysisProfile: profile, - rawResult: execution.value.rawResult, - limitations: execution.value.limitations, - ...(execution.value.relationshipCoverage === undefined - ? {} - : { - relationshipCoverage: execution.value.relationshipCoverage, - }), - locations: execution.value.locations, + subject: + execution.value.subject === undefined + ? active.target + : execution.value.subject, }, + "observed", ); + const sourcesRecorded = recordEvidenceSources( + (source) => this.recordEvidence(source), + execution.value.sourceEvidence ?? [], + ); + if (!sourcesRecorded.ok) return sourcesRecorded; const recorded = this.recordEvidence(evidence); if (!recorded.ok) return recorded; const retained = { ...execution.value, evidenceId: evidence.evidence_id }; diff --git a/src/server/sessionEvidence.ts b/src/application/investigation/recordEvidenceSources.ts similarity index 55% rename from src/server/sessionEvidence.ts rename to src/application/investigation/recordEvidenceSources.ts index a450f31afe..05e7f2ff54 100644 --- a/src/server/sessionEvidence.ts +++ b/src/application/investigation/recordEvidenceSources.ts @@ -1,10 +1,10 @@ -import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; -import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; -import type { Evidence } from "../domain/evidence.js"; -import { ok, type Result } from "../domain/result.js"; +import type { EvidenceWriter } from "./InvestigationRecordPort.js"; +import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; +import type { Evidence } from "../../domain/evidence.js"; +import { ok, type Result } from "../../domain/result.js"; /** Record source evidence in order, stopping at the first session rejection. */ -export const recordSessionEvidenceSources = ( +export const recordEvidenceSources = ( recordEvidence: EvidenceWriter["recordEvidence"] | undefined, sources: readonly Evidence[], ): Result => { diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index 8c7f43a80d..e6e5f04336 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -41,7 +41,10 @@ import { artifactCapabilities } from "./ArtifactProviderMetadata.js"; import { ARTIFACT_GRAPH_PROVIDER } from "../domain/investigationProviders.js"; import { createEvidence } from "../domain/evidence.js"; import { freezeJsonSnapshot } from "../domain/immutableJson.js"; -import { createArtifactInspection } from "../domain/artifactInspection.js"; +import { + artifactInspectionSources, + createArtifactInspection, +} from "../domain/artifactInspection.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import type { ResolvedAppBundleResourceRoot } from "../application/AppBundleExecutable.js"; import type { BundleResourceRoot } from "../domain/apple/bundleResourceRoot.js"; @@ -472,6 +475,7 @@ class ArtifactClient implements AnalysisClient { createAnalysisExecution(result, ARTIFACT_GRAPH_PROVIDER, { rawResult: null, limitations: result.limitations, + sourceEvidence: artifactInspectionSources(result), subject, locations: [], }), diff --git a/src/domain/analysisObservation.ts b/src/domain/analysisObservation.ts index 51d8154f15..06ed145a01 100644 --- a/src/domain/analysisObservation.ts +++ b/src/domain/analysisObservation.ts @@ -20,6 +20,8 @@ export interface AnalysisObservation { /** Provider-owned completeness assessment for caller/callee observations. */ readonly relationshipCoverage?: RelationshipCoverage; readonly locations: readonly EvidenceLocation[]; + /** Atomic source records supplied by the producer, with links derived from their identities. */ + readonly sourceEvidence?: readonly Evidence[]; /** * Missing means the selected operation's default subject should be bound by * its caller. Explicit null means the observation has no file subject. @@ -53,6 +55,8 @@ export const createAnalysisExecutionEvidence = ( ? {} : { relationshipCoverage: execution.relationshipCoverage }), locations: execution.locations, + evidenceLinks: + execution.sourceEvidence?.map(({ evidence_id }) => evidence_id) ?? [], confidence, }, options, diff --git a/src/domain/artifactInspection.ts b/src/domain/artifactInspection.ts index 23ec14742a..c0d99083a3 100644 --- a/src/domain/artifactInspection.ts +++ b/src/domain/artifactInspection.ts @@ -464,7 +464,21 @@ const inspectionCoverage = ( return "complete-within-substeps"; }; -/** Own the atomic source closure published or retained with an inspection. */ +/** Select source records from a typed inspection, preserving sealed child ownership. */ +export const artifactInspectionSources = ( + inspection: ArtifactInspectionResult, +): readonly Evidence[] => { + const substeps = immutableJsonSnapshotMember(inspection, "substeps"); + return Object.freeze( + inspection.substeps.map((step, index) => { + const ownedStep = immutableJsonSnapshotMember(substeps, String(index)); + immutableJsonSnapshotMember(ownedStep, "evidence"); + return step.evidence; + }), + ); +}; + +/** Validate an unknown inspection before selecting its atomic source closure. */ export const readArtifactInspectionSources = ( result: unknown, ): Result => { @@ -473,12 +487,5 @@ export const readArtifactInspectionSources = ( const inspection = canBorrowParsedJsonSnapshot(result, parsed.value) ? result : parsed.value; - const substeps = immutableJsonSnapshotMember(inspection, "substeps"); - return ok( - inspection.substeps.map((step, index) => { - const ownedStep = immutableJsonSnapshotMember(substeps, String(index)); - immutableJsonSnapshotMember(ownedStep, "evidence"); - return step.evidence; - }), - ); + return ok(artifactInspectionSources(inspection)); }; diff --git a/src/server/registerApplicationTools/compareExportShapes.ts b/src/server/registerApplicationTools/compareExportShapes.ts index 5b73070424..81123d7b1e 100644 --- a/src/server/registerApplicationTools/compareExportShapes.ts +++ b/src/server/registerApplicationTools/compareExportShapes.ts @@ -1,6 +1,6 @@ import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { compareJavaScriptExportShapesEvidence } from "../../application/javascript/JavaScriptApplicationWorkflowService.js"; import { toolContract } from "../../contracts/toolContracts.js"; @@ -39,7 +39,7 @@ export const registerCompareJavaScriptExportShapesTool = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, contract); - const recorded = recordSessionEvidenceSources( + const recorded = recordEvidenceSources( options.recordEvidence, result.value.sources, ); diff --git a/src/server/registerApplicationTools/compareSourceToBundle.ts b/src/server/registerApplicationTools/compareSourceToBundle.ts index 1bc6c6a647..2f2ea545dc 100644 --- a/src/server/registerApplicationTools/compareSourceToBundle.ts +++ b/src/server/registerApplicationTools/compareSourceToBundle.ts @@ -9,7 +9,7 @@ import { import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; import { resolveApplicationEvidenceRequest } from "../../application/EvidenceInputResolver.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { compareSourceToBundleEvidence } from "../../application/javascript/JavaScriptApplicationWorkflowService.js"; import { toolContract } from "../../contracts/toolContracts.js"; @@ -96,7 +96,7 @@ export const registerCompareSourceToBundleTool = ( ); if (!selected.ok) return server.delivery.toCallToolResult(selected, contract); - const recorded = recordSessionEvidenceSources( + const recorded = recordEvidenceSources( options.recordEvidence, result.value.sources, ); diff --git a/src/server/registerApplicationTools/compareVersions.ts b/src/server/registerApplicationTools/compareVersions.ts index c98acc7b79..d2a5580dda 100644 --- a/src/server/registerApplicationTools/compareVersions.ts +++ b/src/server/registerApplicationTools/compareVersions.ts @@ -1,6 +1,6 @@ import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { compareApplicationVersionsEvidence } from "../../application/javascript/JavaScriptApplicationWorkflowService.js"; import { toolContract } from "../../contracts/toolContracts.js"; @@ -36,7 +36,7 @@ export const registerCompareApplicationVersionsTool = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, compareContract); - const recorded = recordSessionEvidenceSources( + const recorded = recordEvidenceSources( options.recordEvidence, result.value.sources, ); diff --git a/src/server/registerApplicationTools/obligationLedger.ts b/src/server/registerApplicationTools/obligationLedger.ts index 9bd9aa9bda..67d02773cc 100644 --- a/src/server/registerApplicationTools/obligationLedger.ts +++ b/src/server/registerApplicationTools/obligationLedger.ts @@ -1,6 +1,6 @@ import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { buildResolvedReconstructionObligationLedgerEvidence } from "../../application/ReconstructionObligationLedgerService.js"; import { resolveReconstructionObligationLedgerRequest } from "../../application/ReconstructionObligationLedgerRequest.js"; @@ -45,7 +45,7 @@ export const registerReconstructionObligationLedgerTool = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, contract); - const recorded = recordSessionEvidenceSources( + const recorded = recordEvidenceSources( options.recordEvidence, resolved.value.sources.bundle.records, ); diff --git a/src/server/registerApplicationTools/projectMobileApplicationGraph.ts b/src/server/registerApplicationTools/projectMobileApplicationGraph.ts index 6f8b2a5056..4ab5b4ff53 100644 --- a/src/server/registerApplicationTools/projectMobileApplicationGraph.ts +++ b/src/server/registerApplicationTools/projectMobileApplicationGraph.ts @@ -10,7 +10,7 @@ import { toolContract } from "../../contracts/toolContracts.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { Result } from "../../domain/result.js"; import type { InventoryProjectionEvidenceResult } from "../../application/InventoryProjectionEvidence.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { logToolExecution } from "../toolLogging.js"; import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; import { recordResult } from "./helpers.js"; @@ -67,7 +67,7 @@ const registerProjection = (context: { ); if (!result.ok) return server.delivery.toCallToolResult(result, contract); - const recordedSources = recordSessionEvidenceSources( + const recordedSources = recordEvidenceSources( options.recordEvidence, result.value.sources, ); diff --git a/src/server/registerApplicationTools/traceFeature.ts b/src/server/registerApplicationTools/traceFeature.ts index 5db4253a35..c808afcebe 100644 --- a/src/server/registerApplicationTools/traceFeature.ts +++ b/src/server/registerApplicationTools/traceFeature.ts @@ -1,6 +1,6 @@ import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { traceApplicationFeatureEvidence } from "../../application/javascript/JavaScriptApplicationWorkflowService.js"; import { toolContract } from "../../contracts/toolContracts.js"; @@ -36,7 +36,7 @@ export const registerTraceFeatureTool = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, traceContract); - const recorded = recordSessionEvidenceSources( + const recorded = recordEvidenceSources( options.recordEvidence, result.value.sources, ); diff --git a/src/server/registerApplicationTools/traceSemantics.ts b/src/server/registerApplicationTools/traceSemantics.ts index 14eae128a9..424a53f60f 100644 --- a/src/server/registerApplicationTools/traceSemantics.ts +++ b/src/server/registerApplicationTools/traceSemantics.ts @@ -1,7 +1,7 @@ import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; import { resolveApplicationEvidenceRequest } from "../../application/EvidenceInputResolver.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { traceJavaScriptSemanticsEvidenceValidated } from "../../application/javascript/JavaScriptSemanticTraceService.js"; import { toolContract } from "../../contracts/toolContracts.js"; @@ -44,10 +44,9 @@ export const registerTraceJavaScriptSemanticsTool = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, contract); - const recorded = recordSessionEvidenceSources( - options.recordEvidence, - [parsed.application], - ); + const recorded = recordEvidenceSources(options.recordEvidence, [ + parsed.application, + ]); if (!recorded.ok) return server.delivery.toCallToolResult(recorded, contract); return recordResult( diff --git a/src/server/registerArtifactComparisonTool.ts b/src/server/registerArtifactComparisonTool.ts index 3715ba7834..3e9fe67a40 100644 --- a/src/server/registerArtifactComparisonTool.ts +++ b/src/server/registerArtifactComparisonTool.ts @@ -14,7 +14,7 @@ import { runDerivedOperation } from "./runDerivedOperation.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import { withAdmittedAnalysis } from "./analysisAdmission.js"; import { runAdmittedToolOperation } from "./admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; /** Register Evidence-backed deterministic artifact comparison. */ export const registerArtifactComparisonTool = ( @@ -60,7 +60,7 @@ export const registerArtifactComparisonTool = ( if (!computed.ok) return server.delivery.toCallToolResult(computed, contract); const { left, right, comparison } = computed.value; - const recordedSources = recordSessionEvidenceSources( + const recordedSources = recordEvidenceSources( (source) => session.recordEvidence(source), [left, right], ); diff --git a/src/server/registerArtifactTools.ts b/src/server/registerArtifactTools.ts index e852654ce7..a29497aa4c 100644 --- a/src/server/registerArtifactTools.ts +++ b/src/server/registerArtifactTools.ts @@ -3,7 +3,6 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import { ARTIFACT_TOOL_CONTRACTS } from "../contracts/artifactToolContracts.js"; import { createArtifactExtractionDestination } from "../application/artifacts/ArtifactExtractionDestination.js"; import { registerEvidenceTools } from "./registerEvidenceTools.js"; -import { readArtifactInspectionSources } from "../domain/artifactInspection.js"; import { retainedSummaryDelivery, withoutDeliveryDetail, @@ -34,11 +33,5 @@ export const registerArtifactTools = ( ? { ...parameters, output_root: createArtifactExtractionDestination() } : executionWithoutDelivery(operation, parameters), parametersForEvidence: decodeWithoutDelivery, - sourceEvidence: (operation, result) => { - if (operation !== "inspect_artifact") return []; - const parsed = readArtifactInspectionSources(result); - if (!parsed.ok) throw parsed.error; - return parsed.value; - }, }); }; diff --git a/src/server/registerBundleComparisonTool.ts b/src/server/registerBundleComparisonTool.ts index 52bb0dd62c..1a90b3e839 100644 --- a/src/server/registerBundleComparisonTool.ts +++ b/src/server/registerBundleComparisonTool.ts @@ -7,7 +7,7 @@ import { compareBundles } from "../domain/bundleComparison.js"; import { createEvidence } from "../domain/evidence.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; -import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; import { BUNDLE_COMPARISON_PROVIDER } from "../domain/investigationProviders.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import { withAdmittedAnalysis } from "./analysisAdmission.js"; @@ -65,7 +65,7 @@ export const registerBundleComparisonTool = ( ]), ).values(), ]; - const recordedSources = recordSessionEvidenceSources( + const recordedSources = recordEvidenceSources( (evidence) => session.recordEvidence(evidence), sourceRecords, ); diff --git a/src/server/registerEvidenceTools.ts b/src/server/registerEvidenceTools.ts index 6f927b315c..59f4910b85 100644 --- a/src/server/registerEvidenceTools.ts +++ b/src/server/registerEvidenceTools.ts @@ -7,7 +7,8 @@ import type { } from "../application/AnalysisProvider.js"; import type { ToolContract } from "../contracts/toolContractTypes.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; -import { createEvidence, type Evidence } from "../domain/evidence.js"; +import type { Evidence } from "../domain/evidence.js"; +import { createAnalysisExecutionEvidence } from "../domain/analysisObservation.js"; import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { ok, type Result } from "../domain/result.js"; @@ -16,7 +17,7 @@ import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; -import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; import { reportOperationStart, reportOperationEnd, @@ -30,10 +31,6 @@ interface EvidenceToolRegistration { readonly analysisForAdmission?: ( analysis: AnalysisOperationPort, ) => AnalysisOperationPort; - readonly sourceEvidence?: ( - operation: Exclude, - result: import("../domain/jsonValue.js").JsonValue, - ) => readonly Evidence[]; readonly parametersForExecution?: ( operation: Exclude, parameters: Readonly>, @@ -94,36 +91,21 @@ export const registerEvidenceTools = ( await reportOperationEnd(progress, contract.name, execution.ok); if (!execution.ok) return server.delivery.toCallToolResult(execution, contract); - const sourceEvidence = - options.sourceEvidence?.(contract.name, execution.value.result) ?? - []; - const evidence = createEvidence( - execution.value.subject === undefined - ? options.activeTarget?.() - : (execution.value.subject ?? undefined), - execution.value.provider, + const sourceEvidence = execution.value.sourceEvidence ?? []; + const evidence = createAnalysisExecutionEvidence( + contract.name, + options.parametersForEvidence?.(contract.name, parameters) ?? + parameters, { - operation: contract.name, - parameters: - options.parametersForEvidence?.(contract.name, parameters) ?? - parameters, - result: execution.value.result, - ...(execution.value.analysisProfile === undefined - ? {} - : { analysisProfile: execution.value.analysisProfile }), - rawResult: execution.value.rawResult, - limitations: execution.value.limitations, - ...(execution.value.relationshipCoverage === undefined - ? {} - : { - relationshipCoverage: - execution.value.relationshipCoverage, - }), - locations: execution.value.locations, - evidenceLinks: sourceEvidence.map(({ evidence_id: id }) => id), + ...execution.value, + subject: + execution.value.subject === undefined + ? (options.activeTarget?.() ?? null) + : execution.value.subject, }, + "observed", ); - const sourceRecorded = recordSessionEvidenceSources( + const sourceRecorded = recordEvidenceSources( options.recordEvidence, sourceEvidence, ); diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts index 462a098bf5..4817c183b5 100644 --- a/src/server/registerFunctionComparisonTool.ts +++ b/src/server/registerFunctionComparisonTool.ts @@ -9,7 +9,7 @@ import { FUNCTION_COMPARISON_PROVIDER } from "../domain/comparisonEvidenceIdenti import { jsonValueSchema } from "../domain/jsonValue.js"; import type { RecordUnknownInput } from "../domain/residualUnknown.js"; import { recordDerivedEvidence } from "./recordDerivedEvidence.js"; -import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import { withAdmittedAnalysis } from "./analysisAdmission.js"; @@ -68,7 +68,7 @@ export const registerFunctionComparisonTool = ( const rightEvidence = right.evidence; const leftIds = [leftEvidence.evidence_id]; const rightIds = [rightEvidence.evidence_id]; - const recordedSources = recordSessionEvidenceSources( + const recordedSources = recordEvidenceSources( (evidence) => session.recordEvidence(evidence), [leftEvidence, rightEvidence], ); diff --git a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts index 0c3f451ff2..5cde172ce1 100644 --- a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts +++ b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts @@ -1,7 +1,7 @@ import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { executeManagedMemberComparison } from "../../application/managed/ManagedMemberComparisonService.js"; import { logToolExecution } from "../toolLogging.js"; @@ -33,7 +33,7 @@ export const registerCompareManagedMembers = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, compareContract); - const recorded = recordSessionEvidenceSources( + const recorded = recordEvidenceSources( options.recordEvidence, result.value.sources, ); diff --git a/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts b/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts index a8690a7b13..7b185fa241 100644 --- a/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts +++ b/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts @@ -1,6 +1,6 @@ import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { projectManagedGraphEvidence } from "../../application/managed/ManagedApplicationGraphService.js"; import { readManagedGraphProjection } from "../../domain/managed/managedApplicationGraph.js"; @@ -39,7 +39,7 @@ export const registerProjectManagedApplicationGraph = ( ); if (!result.ok) return server.delivery.toCallToolResult(result, graphContract); - const recordedSources = recordSessionEvidenceSources( + const recordedSources = recordEvidenceSources( options.recordEvidence, [ request.value.sources.artifact?.evidence, diff --git a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts index 23f6ca7d42..d420c22dad 100644 --- a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts +++ b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts @@ -1,7 +1,7 @@ import { workflowUnknownInput } from "../../application/investigation/WorkflowUnknown.js"; import type { EvidenceMcpServer } from "../EvidenceMcpServer.js"; import { runAdmittedToolOperation } from "../admittedToolOperation.js"; -import { recordSessionEvidenceSources } from "../sessionEvidence.js"; +import { recordEvidenceSources } from "../../application/investigation/recordEvidenceSources.js"; import { executeManagedNativeBoundaries } from "../../application/managed/ManagedNativeVerificationService.js"; import { logToolExecution } from "../toolLogging.js"; @@ -39,7 +39,7 @@ export const registerVerifyManagedNativeBoundaries = ( nativeVerificationContract, ); const execution = result.value; - const recordedSources = recordSessionEvidenceSources( + const recordedSources = recordEvidenceSources( options.recordEvidence, execution.sourceEvidence, ); diff --git a/src/server/registerNativeFunctionMatchingTool.ts b/src/server/registerNativeFunctionMatchingTool.ts index cd9e783a57..2a80c2734f 100644 --- a/src/server/registerNativeFunctionMatchingTool.ts +++ b/src/server/registerNativeFunctionMatchingTool.ts @@ -6,7 +6,7 @@ import { summarizeRetainedAnalysis } from "../application/analysisView/AnalysisV import { withAdmittedAnalysis } from "./analysisAdmission.js"; import { createEvidenceToolHandler } from "./contractToolHandler.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; import { ok } from "../domain/result.js"; import type { Logger } from "pino"; @@ -25,7 +25,7 @@ export const registerNativeFunctionMatchingTool = ( async (input, options) => { const result = await matchNativeFunctions(input, options, evidenceById); if (!result.ok) return result; - const recorded = recordSessionEvidenceSources( + const recorded = recordEvidenceSources( recordEvidence, result.value.sources, ); diff --git a/src/server/registerProcessComparisonTool.ts b/src/server/registerProcessComparisonTool.ts index 956654f3a6..b711860a9d 100644 --- a/src/server/registerProcessComparisonTool.ts +++ b/src/server/registerProcessComparisonTool.ts @@ -18,7 +18,7 @@ import { parseProcessCaptureEvidence } from "../application/process/ProcessEvide import type { RecordUnknownInput } from "../domain/residualUnknown.js"; import { err, type Result } from "../domain/result.js"; import { recordDerivedEvidence } from "./recordDerivedEvidence.js"; -import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; import { PROCESS_PROVIDER } from "../domain/process/processEvidenceProvider.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; @@ -120,7 +120,7 @@ export const registerProcessComparisonTool = ( if (!computed.ok) return server.delivery.toCallToolResult(computed, contract); const { comparison, evidence } = computed.value; - const recordedSources = recordSessionEvidenceSources( + const recordedSources = recordEvidenceSources( (evidence) => session.recordEvidence(evidence), [leftRecord, rightRecord], ); diff --git a/src/server/registerWasmTools.ts b/src/server/registerWasmTools.ts index 59e0a48e4e..b929e759c0 100644 --- a/src/server/registerWasmTools.ts +++ b/src/server/registerWasmTools.ts @@ -1,7 +1,7 @@ import { compareWasmInterfaces } from "../application/wasm/WasmInterfaceComparisonService.js"; import { COMPARE_WASM_INTERFACES_CONTRACT } from "../contracts/wasm/wasmComparisonContracts.js"; import { resolvePairedEvidenceRequest } from "../application/EvidenceInputResolver.js"; -import { recordSessionEvidenceSources } from "./sessionEvidence.js"; +import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { WasmArtifactService } from "../application/wasm/WasmArtifactService.js"; @@ -47,7 +47,7 @@ export const registerWasmTools = ( if (!sources.ok) return sources; const result = await compareWasmInterfaces(sources.value, options); if (!result.ok) return result; - const recorded = recordSessionEvidenceSources(recordEvidence, [ + const recorded = recordEvidenceSources(recordEvidence, [ sources.value.left, sources.value.right, ]); diff --git a/tests/boundary/mcp/artifactMcp.test.ts b/tests/boundary/mcp/artifactMcp.test.ts index 74abc4a027..2227cec5df 100644 --- a/tests/boundary/mcp/artifactMcp.test.ts +++ b/tests/boundary/mcp/artifactMcp.test.ts @@ -5,7 +5,13 @@ import { } from "../../../src/contracts/investigationExamples.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../../src/domain/evidence.js"; +import { EvidenceIntegrityError } from "../../../src/domain/evidenceErrors.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; import { @@ -15,14 +21,17 @@ import { occurrenceIdForLocation, } from "../../../src/domain/artifactIdentity.js"; import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; -import { ok as resultOk } from "../../../src/domain/result.js"; +import { + ok as resultOk, + err as resultErr, +} from "../../../src/domain/result.js"; import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { createPackageWithOptions } from "@electron/asar"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; -import { expect, it } from "vitest"; +import { expect, it, vi, type MockInstance } from "vitest"; import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -35,6 +44,7 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js"; import { createServer } from "../../../src/server/createServer.js"; +import { createArtifactBinarySession } from "../../../src/composition/binary.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { cArchiveExecutable } from "../../../src/artifacts/pyinstaller/CArchive.fixture.js"; import { compareUnicodeCodePoints } from "../../../src/domain/unicodeCodePointOrder.js"; @@ -46,6 +56,80 @@ const integrityMessage = (unpacked: boolean): string => : "Artifact bytes contradict declared integrity." } If expected, rerun inspect_artifact with integrity_policy=record-and-continue (CLI: --integrity-policy record-and-continue) to retain observed bytes as untrusted; otherwise get a fresh copy.`; +it("refuses an artifact parent and summary when its actual inventory source cannot be recorded", async () => { + const root = await createTestTempDirectory( + "rea-artifact-source-refusal-mcp-", + ); + const archive = join(root, "fixture.zip"); + const zip = new ZipWriter(new Uint8ArrayWriter()); + await zip.add("main.js", new TextReader("console.log('source');\n")); + await writeFile(archive, await zip.close()); + const session = await createArtifactBinarySession({}); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "artifact-source-refusal", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const attempted: Evidence[] = []; + const failure = new EvidenceIntegrityError( + "controlled inventory source recording refusal", + ); + const nativeRecord = session.recordEvidence.bind(session); + let recording: MockInstance | undefined; + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: archive }, + }); + expect(opened.isError).not.toBe(true); + const before = session.exportEvidenceBundle().records; + recording = vi + .spyOn(session, "recordEvidence") + .mockImplementation((evidence) => { + attempted.push(evidence); + return evidence.operation === "inventory_artifact" + ? resultErr(failure) + : nativeRecord(evidence); + }); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: { detail: "summary" }, + }); + expect(inspected.isError).toBe(true); + expect(parseMcpToolError(inspected)).toMatchObject({ + error: projectAnalysisError(failure), + }); + expect(attempted.map(({ operation }) => operation)).toEqual([ + "inventory_artifact", + ]); + expect(attempted[0]?.normalized_result).toMatchObject({ + occurrences: expect.arrayContaining([ + expect.objectContaining({ logical_path: "main.js" }), + ]), + }); + expect(session.exportEvidenceBundle().records).toEqual(before); + } finally { + recording?.mockRestore(); + const closed = await Promise.allSettled([ + client.close(), + server.close(), + session.close(), + ]); + const failures: unknown[] = []; + for (const result of closed) + if (result.status === "rejected") failures.push(result.reason); + const sessionClosed = closed[2]; + if (sessionClosed.status === "fulfilled" && !sessionClosed.value.ok) + failures.push(sessionClosed.value.error); + if (failures.length > 0) + throw new AggregateError( + failures, + "Artifact recorder refusal cleanup failed", + ); + } +}); + it.each([ { unpacked: false, label: "embedded" }, { unpacked: true, label: "unpacked" }, diff --git a/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts b/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts index 1cb7b9239b..9f032b327c 100644 --- a/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts +++ b/tests/boundary/providers/native/artifactProvider.directAnalysis.test.ts @@ -10,6 +10,8 @@ import { createArtifactBinarySession } from "../../../../src/composition/binary. import { thinMach } from "../../../../src/domain/binaryTarget.fixture.js"; import { interfaceBuilderAnalysisSchema } from "../../../../src/domain/apple/interfaceBuilderGraph.js"; import { artifactInspectionResultSchema } from "../../../../src/domain/artifactInspection.js"; +import { createAnalysisExecutionEvidence } from "../../../../src/domain/analysisObservation.js"; +import { immutableJsonSnapshotMember } from "../../../../src/domain/immutableJson.js"; import { parseEvidence } from "../../../../src/domain/evidence.js"; const { runProviderAnalysis } = createDirectAnalysis({}); @@ -75,4 +77,44 @@ it("inspects an artifact through the direct CLI workflow without an REA grant", expect.objectContaining({ logical_path: "main.js" }), ]), }); + + const staticSession = await createArtifactBinarySession({}); + try { + const opened = await staticSession.open(archive); + if (!opened.ok) throw opened.error; + const executed = await staticSession.execute("inspect_artifact", {}); + if (!executed.ok) throw executed.error; + const sessionInspection = artifactInspectionResultSchema.parse( + executed.value.result, + ); + expect(sessionInspection.evidence_links).toEqual( + sessionInspection.substeps.map( + ({ evidence: sourceEvidence }) => sourceEvidence.evidence_id, + ), + ); + const substeps = immutableJsonSnapshotMember( + executed.value.result, + "substeps", + ); + const sources = executed.value.sourceEvidence; + expect(sources).toHaveLength(sessionInspection.substeps.length); + if (sources === undefined) + throw new Error("Missing committed inspection sources"); + for (const [index, sourceEvidence] of sources.entries()) { + const step = immutableJsonSnapshotMember(substeps, String(index)); + expect(sourceEvidence).toBe( + immutableJsonSnapshotMember(step, "evidence"), + ); + } + const parent = createAnalysisExecutionEvidence( + "inspect_artifact", + {}, + executed.value, + "observed", + ); + expect(parent.evidence_links).toEqual(sessionInspection.evidence_links); + } finally { + const closed = await staticSession.close(); + if (!closed.ok) throw closed.error; + } }); diff --git a/tests/composition/directAnalysisCleanup.test.ts b/tests/composition/directAnalysisCleanup.test.ts index 10389c2399..173d0aedab 100644 --- a/tests/composition/directAnalysisCleanup.test.ts +++ b/tests/composition/directAnalysisCleanup.test.ts @@ -1,6 +1,8 @@ -import { readFile, readdir, stat } from "node:fs/promises"; +import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { createPackageWithOptions } from "@electron/asar"; +import { createArtifactBinarySession } from "../../src/composition/binary.js"; import { runDirectAnalysis, runManagedProviderExecution, @@ -19,7 +21,9 @@ import { projectAnalysisError, projectAnalysisErrorWithoutCapturedOutput, } from "../../src/domain/analysisErrorProjection.js"; -import { parseEvidence } from "../../src/domain/evidence.js"; +import { parseEvidence, type Evidence } from "../../src/domain/evidence.js"; +import { jsonObjectSchema } from "../../src/domain/jsonValue.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; import { ProviderCleanupError } from "../../src/domain/providerCleanupError.js"; import { ghidraFunctionDossier } from "../../src/domain/ghidraValues.fixture.js"; import { @@ -36,6 +40,7 @@ import { createBinarySessionTargets, createTestBinarySession, } from "../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; const cleanupError = new ProviderCleanupError( "fixture", @@ -100,6 +105,94 @@ const factories = ( return { dependencies, session }; }; +it.each([false, true])( + "retains actual artifact sources and caller path after final-close cancellation (cleanup fails: %s)", + async (closeFails) => { + const root = await createTestTempDirectory("rea-artifact-final-close-"); + const source = join(root, "source"); + const archive = join(root, "fixture.asar"); + await mkdir(source); + await writeFile(join(source, "main.js"), "console.log('retained');\n"); + await createPackageWithOptions(source, archive, {}); + const session = await createArtifactBinarySession({}); + const nativeClose = session.close.bind(session); + const controller = new AbortController(); + const reason = { + request: "cancel during actual artifact session final close", + }; + let completed: Evidence | undefined; + const close = vi + .spyOn(session, "close") + .mockImplementationOnce(async () => { + completed = session + .exportEvidenceBundle() + .records.find( + ({ operation, caller_input_path }) => + operation === "inspect_artifact" && caller_input_path === archive, + ); + controller.abort(reason); + // A controlled refusal retains the actual static session for finally + // cleanup; successful cleanup runs its native owner here. + return closeFails ? err(cleanupError) : nativeClose(); + }); + const unexpectedSession = () => { + throw new Error("Artifact one-shot must select its static session"); + }; + const dependencies: DirectAnalysisDependencies = { + readConfiguration: () => parseConfig({}), + createArtifactBinarySession: () => session, + createBinarySession: unexpectedSession, + createNativeDispatchBinarySession: unexpectedSession, + createNativeBinarySession: unexpectedSession, + createManagedBinarySession: unexpectedSession, + }; + try { + const result = analysisCliErrorEnvelopeSchema.parse( + await runProviderAnalysis( + dependencies, + archive, + "inspect_artifact", + {}, + undefined, + controller.signal, + ), + ); + expect(result.code).toBe(closeFails ? "cleanup_incomplete" : "cancelled"); + expect(result.category).toBe("cancelled"); + expect(completed).toBeDefined(); + const details = jsonObjectSchema.parse(result.details); + const partial = parseEvidence(details.partial_observation); + expect(partial).toEqual(completed); + expect(partial.operation).toBe("inspect_artifact"); + expect(partial.caller_input_path).toBe(archive); + expect(partial.subject?.local_path).toBe(archive); + const inspection = artifactInspectionResultSchema.parse( + partial.normalized_result, + ); + expect(inspection.subject.root_format).toBe("asar"); + expect(partial.evidence_links).toEqual(inspection.evidence_links); + expect(partial.evidence_links).toEqual( + inspection.substeps.map(({ evidence }) => evidence.evidence_id), + ); + expect(inspection.substeps[0]?.evidence.normalized_result).toMatchObject({ + occurrences: expect.arrayContaining([ + expect.objectContaining({ logical_path: "main.js" }), + ]), + }); + if (closeFails) + expect(details.cleanup_error).toEqual( + projectAnalysisError(cleanupError), + ); + else expect(result.details).not.toHaveProperty("cleanup_error"); + expect((await readdir(root)).sort()).toEqual(["fixture.asar", "source"]); + } finally { + close.mockRestore(); + const closed = await nativeClose(); + if (!closed.ok) throw closed.error; + } + }, +); + describe("one-shot finalization cancellation", () => { it.each(["read_bytes", "analyze_function"] as const)( "retains completed %s Evidence when cancellation arrives during final close", From 58fb8241055b326d9dc40652e6859893587c7a86 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:01:36 +0800 Subject: [PATCH 13/37] refactor(evidence): centralize JSON payload admission Give ordinary Evidence creation an explicit unknown-payload input contract while retaining the strong JSON contract for producer helpers and cooperative immutable creation. Keep schema admission, input snapshotting and identity hashing with the Evidence owner. Remove 32 generic JSON preparses that only fed ordinary Evidence construction. Preserve semantic format validation, selected-source bindings, standalone JSON boundaries, typed executions and scenario digest contracts. Invalid JSON now reports its Evidence parameters/normalized_result/raw_result prefix. Validation: fresh runtime/catalog build, 560 focused and README tests passed, 4 platform-specific cases skipped, full static/docs checks, test discovery, duplication and diff checks. New regressions cover omitted semantic optionals, raw-result defaults and nested undefined refusal without input mutation. Installed providers, real browsers/Frida/JADX and Linux-only recovery remain unverified; boundary providers are authored fixtures. --- docs/tool-design.md | 11 +++ src/application/BrowserEvidence.ts | 3 +- src/application/BrowserScenarioEvidence.ts | 4 +- .../ReconstructionObligationLedger.fixture.ts | 15 +-- src/application/WebModuleTraceService.ts | 2 +- src/application/WebRuntimeService.ts | 3 +- src/application/WebScriptExportService.ts | 3 +- src/application/WebSourceLocationService.ts | 3 +- .../analysisView/AnalysisViewService.ts | 5 +- .../android/AndroidAnalysisService.ts | 3 +- .../frida/FridaInstrumentationService.ts | 5 +- .../javascript/ElectronActiveEvidence.ts | 4 +- .../javascript/JavaScriptRecoveryService.ts | 5 +- .../JavaScriptRuntimeObservationEvidence.ts | 3 +- src/application/process/ProcessEvidence.ts | 3 +- .../sqlite/SqliteDatabaseService.ts | 3 +- src/contracts/investigationExamples.ts | 5 +- src/domain/analysisObservation.ts | 4 +- src/domain/evidence.test.ts | 96 +++++++++++++++++++ src/domain/evidence.ts | 24 +++-- src/domain/pcap/pcapEvidence.ts | 4 +- .../process/processComparisonEvidence.ts | 5 +- src/server/registerArtifactComparisonTool.ts | 3 +- src/server/registerBundleComparisonTool.ts | 3 +- src/server/registerFunctionComparisonTool.ts | 3 +- src/server/registerInvestigationTools.ts | 9 +- 26 files changed, 162 insertions(+), 69 deletions(-) diff --git a/docs/tool-design.md b/docs/tool-design.md index 5c2d3b7391..1fb7e2d901 100644 --- a/docs/tool-design.md +++ b/docs/tool-design.md @@ -241,6 +241,17 @@ Put provider protocols/parsing in adapters and shared CLI/MCP workflows in the application layer. Share Evidence provenance, unknown projection, and session retention semantics; matching payloads alone do not establish parity. +Let `createEvidence` own JSON validation and snapshotting of ordinary observation +payloads. `EvidenceObservationInput` accepts semantically parsed parameters, +results, and raw results whose JSON validity still needs admission. Keep format, +selection, and identity validation at their producing boundary. Invalid JSON +issues are rooted at `parameters`, `normalized_result`, or `raw_result` in the +Evidence envelope. + +`EvidenceObservation` retains the stronger JSON contract used by producer +helpers and `createImmutableEvidenceSteps`. The cooperative constructor also +requires an authenticated immutable result; preserve that ownership proof. + Measure the resource before choosing a budget and enforce it before the costly step. Account for value alternatives, trace-frame products, retained records, serialization expansion, and temporary-file growth separately. Derive facet diff --git a/src/application/BrowserEvidence.ts b/src/application/BrowserEvidence.ts index 311d8ac549..a704a6b0e0 100644 --- a/src/application/BrowserEvidence.ts +++ b/src/application/BrowserEvidence.ts @@ -4,7 +4,6 @@ import { type Evidence, type EvidenceObservation, } from "../domain/evidence.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; import type { InspectWebPageInput, @@ -77,7 +76,7 @@ export const createBrowserEvidence = ( predicateType: browserPredicate(operation), operation, parameters: browserParameters(input, result), - result: jsonValueSchema.parse(result), + result, confidence: "observed", authority: "external-service", environment: diff --git a/src/application/BrowserScenarioEvidence.ts b/src/application/BrowserScenarioEvidence.ts index 5dae07c218..7fd9c2bd2e 100644 --- a/src/application/BrowserScenarioEvidence.ts +++ b/src/application/BrowserScenarioEvidence.ts @@ -7,7 +7,7 @@ import { type Evidence, type EvidenceObservation, } from "../domain/evidence.js"; -import { jsonObjectSchema, jsonValueSchema } from "../domain/jsonValue.js"; +import { jsonObjectSchema } from "../domain/jsonValue.js"; const browserScenarioParameters = ( scenario: BrowserScenario, @@ -26,7 +26,7 @@ export const createBrowserScenarioEvidence = ( predicateType: "rea.browser-scenario-capture", operation: "capture_browser_scenario", parameters: browserScenarioParameters(scenario), - result: jsonValueSchema.parse(capture), + result: capture, confidence: "observed", authority: "controlled-replay", environment: { diff --git a/src/application/ReconstructionObligationLedger.fixture.ts b/src/application/ReconstructionObligationLedger.fixture.ts index ca1da93d09..5d2e422592 100644 --- a/src/application/ReconstructionObligationLedger.fixture.ts +++ b/src/application/ReconstructionObligationLedger.fixture.ts @@ -12,7 +12,6 @@ import { createEvidenceBundle, type EvidenceBundle, } from "../domain/evidenceBundle.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { reconstructionObligationLedgerSchema, type ReconstructionObligationLedgerInput, @@ -47,12 +46,10 @@ export const processEvidence = ( predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, - result: jsonValueSchema.parse( - processCaptureSchema.parse({ - ...EMPTY_PROCESS_CAPTURE_EXAMPLE, - ...overrides, - }), - ), + result: processCaptureSchema.parse({ + ...EMPTY_PROCESS_CAPTURE_EXAMPLE, + ...overrides, + }), confidence: "observed", authority: "controlled-replay", }); @@ -70,9 +67,7 @@ export const boundProofEvidence = ( predicateType: "rea.reconstruction-proof", operation: "verify_reconstruction_obligations", parameters: {}, - result: jsonValueSchema.parse( - reconstructionObligationProofSchema.parse(result), - ), + result: reconstructionObligationProofSchema.parse(result), confidence: "observed", authority, }, diff --git a/src/application/WebModuleTraceService.ts b/src/application/WebModuleTraceService.ts index b5f5878a67..ee57889e7a 100644 --- a/src/application/WebModuleTraceService.ts +++ b/src/application/WebModuleTraceService.ts @@ -261,7 +261,7 @@ export class WebModuleTraceService { }, { operation: OPERATION, - parameters: jsonObjectSchema.parse(input), + parameters: input, result: normalized.data, rawResult: resolution?.rawResult ?? null, confidence: "derived", diff --git a/src/application/WebRuntimeService.ts b/src/application/WebRuntimeService.ts index acdacb0f32..64dd8d4d3b 100644 --- a/src/application/WebRuntimeService.ts +++ b/src/application/WebRuntimeService.ts @@ -7,7 +7,6 @@ import { AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import { createEvidence, type Evidence } from "../domain/evidence.js"; -import { jsonObjectSchema } from "../domain/jsonValue.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -162,7 +161,7 @@ export class WebRuntimeService { createEvidence(undefined, this.provider.identity(), { predicateType: `rea.${operation.replaceAll("_", "-")}`, operation, - parameters: jsonObjectSchema.parse(input), + parameters: input, result, confidence: "observed", authority: "external-service", diff --git a/src/application/WebScriptExportService.ts b/src/application/WebScriptExportService.ts index c81617304f..c0a52b351a 100644 --- a/src/application/WebScriptExportService.ts +++ b/src/application/WebScriptExportService.ts @@ -21,7 +21,6 @@ import { AnalysisError } from "../domain/analysisErrorBase.js"; import { createEvidence, type Evidence } from "../domain/evidence.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -136,7 +135,7 @@ export const exportWebScriptsValidated = async ( capture_path: input.capture_path, output_directory: input.output_directory, }, - result: jsonValueSchema.parse(result), + result, rawResult: null, confidence: "derived", authority: "historical-reference", diff --git a/src/application/WebSourceLocationService.ts b/src/application/WebSourceLocationService.ts index b40ad21870..1a5ad8325b 100644 --- a/src/application/WebSourceLocationService.ts +++ b/src/application/WebSourceLocationService.ts @@ -9,7 +9,6 @@ import { } from "../domain/analysisErrorCore.js"; import { createEvidence, type Evidence } from "../domain/evidence.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; -import { jsonObjectSchema } from "../domain/jsonValue.js"; import { err, ok, type Result } from "../domain/result.js"; import { selectedWebScriptArtifactsSchema, @@ -171,7 +170,7 @@ export class WebSourceLocationService { }, { operation: OPERATION, - parameters: jsonObjectSchema.parse(input), + parameters: input, result: normalized.data, rawResult: { source_map_text: map.text }, confidence: "derived", diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index 2d83a52a0b..55e614be31 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -2,7 +2,6 @@ import { setImmediate } from "node:timers/promises"; import { dispatchQueuedInterrupts } from "../../process/queuedInterrupts.js"; import { readImmutableEvidenceBundle } from "../EvidenceBundleFiles.js"; import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../../domain/analysisView/analysisViewErrors.js"; -import { jsonValueSchema } from "../../domain/jsonValue.js"; import { ANALYSIS_VIEW_PROVIDER } from "../../domain/investigationProviders.js"; import type { EvidenceLookup } from "../EvidenceInputResolver.js"; @@ -346,9 +345,9 @@ const createAnalysisViewEvidence = ( operation: INSPECT_ANALYSIS_VIEW_OPERATION, parameters: { parent_evidence_id: parent.evidence_id, - view: jsonValueSchema.parse(input.view), + view: input.view, }, - result: jsonValueSchema.parse(result), + result, rawResult: null, confidence: // Grouping is whole-capture, so any templated group or parameter makes the result inferred; diff --git a/src/application/android/AndroidAnalysisService.ts b/src/application/android/AndroidAnalysisService.ts index f31dff0dc7..12433b3311 100644 --- a/src/application/android/AndroidAnalysisService.ts +++ b/src/application/android/AndroidAnalysisService.ts @@ -11,7 +11,6 @@ import { import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; -import { jsonObjectSchema } from "../../domain/jsonValue.js"; import { err, type Result } from "../../domain/result.js"; import { completeProviderEvidence } from "../ProviderOperationExecution.js"; @@ -62,7 +61,7 @@ export class AndroidAnalysisService { execution.provider, { operation, - parameters: jsonObjectSchema.parse(request.data.input), + parameters: request.data.input, result: execution.result, rawResult: execution.rawResult, limitations: execution.limitations, diff --git a/src/application/frida/FridaInstrumentationService.ts b/src/application/frida/FridaInstrumentationService.ts index 685347bd30..b0f52f7c67 100644 --- a/src/application/frida/FridaInstrumentationService.ts +++ b/src/application/frida/FridaInstrumentationService.ts @@ -3,7 +3,6 @@ import { access, stat } from "node:fs/promises"; import { setTimeout as delay } from "node:timers/promises"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; -import { jsonObjectSchema, jsonValueSchema } from "../../domain/jsonValue.js"; import { resolvedAnalysisError, type AnalysisError, @@ -458,8 +457,8 @@ const fridaEvidence = ( { predicateType: "rea.frida.instrumentation-observation", operation, - parameters: jsonObjectSchema.parse(parameters), - result: jsonValueSchema.parse(result), + parameters, + result, ...(script?.sourceRepresentation === undefined ? {} : { diff --git a/src/application/javascript/ElectronActiveEvidence.ts b/src/application/javascript/ElectronActiveEvidence.ts index cbd844e263..ebf1569cda 100644 --- a/src/application/javascript/ElectronActiveEvidence.ts +++ b/src/application/javascript/ElectronActiveEvidence.ts @@ -4,7 +4,7 @@ import type { CanonicalElectronActiveObservationInput } from "./ElectronActiveOb import type { Evidence, EvidenceObservation } from "../../domain/evidence.js"; import { createEvidence } from "../../domain/evidence.js"; import type { ElectronActiveObservationResult } from "../../domain/javascript/electronActiveObservation.js"; -import { jsonObjectSchema, jsonValueSchema } from "../../domain/jsonValue.js"; +import { jsonObjectSchema } from "../../domain/jsonValue.js"; /** Commit the selected Electron experiment and its observed result. */ export const createElectronActiveEvidence = ( @@ -16,7 +16,7 @@ export const createElectronActiveEvidence = ( predicateType: "rea.electron-active-scenario", operation: "capture_electron_scenario", parameters: parameters(input), - result: jsonValueSchema.parse(result), + result, confidence: "observed", authority: "controlled-replay", environment: { diff --git a/src/application/javascript/JavaScriptRecoveryService.ts b/src/application/javascript/JavaScriptRecoveryService.ts index 5fa5624a9c..c5291f148d 100644 --- a/src/application/javascript/JavaScriptRecoveryService.ts +++ b/src/application/javascript/JavaScriptRecoveryService.ts @@ -11,7 +11,6 @@ import { javascriptRecoveryInputSchema, javascriptRecoveryResultSchema, } from "../../domain/javascript/javascriptRecovery.js"; -import { jsonObjectSchema, jsonValueSchema } from "../../domain/jsonValue.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { err, type Result } from "../../domain/result.js"; import type { ExecutionOptions } from "../AnalysisProvider.js"; @@ -115,8 +114,8 @@ export class JavaScriptRecoveryService { execution.value.provider, { operation: OPERATION, - parameters: jsonObjectSchema.parse(parsed.data), - result: jsonValueSchema.parse(result.data), + parameters: parsed.data, + result: result.data, rawResult: execution.value.rawResult, locations: execution.value.locations, limitations: execution.value.limitations, diff --git a/src/application/javascript/JavaScriptRuntimeObservationEvidence.ts b/src/application/javascript/JavaScriptRuntimeObservationEvidence.ts index a5f0d9a2a8..399f20b3f1 100644 --- a/src/application/javascript/JavaScriptRuntimeObservationEvidence.ts +++ b/src/application/javascript/JavaScriptRuntimeObservationEvidence.ts @@ -10,7 +10,6 @@ import type { ListJavaScriptRuntimeTargetsInput, ObserveJavaScriptRuntimeInput, } from "../../domain/javascript/javascriptRuntimeObservation.js"; -import { jsonValueSchema } from "../../domain/jsonValue.js"; type RuntimeObservationOperation = | "list_javascript_runtime_targets" @@ -30,7 +29,7 @@ export const createJavaScriptRuntimeObservationEvidence = ( : "rea.javascript-runtime-observation", operation, parameters: parameters(input), - result: jsonValueSchema.parse(result), + result, rawResult: null, confidence: "observed", authority: "external-service", diff --git a/src/application/process/ProcessEvidence.ts b/src/application/process/ProcessEvidence.ts index cee402e545..4fd233088a 100644 --- a/src/application/process/ProcessEvidence.ts +++ b/src/application/process/ProcessEvidence.ts @@ -10,7 +10,6 @@ import { PROCESS_PROVIDER, isProcessEvidenceProvider, } from "../../domain/process/processEvidenceProvider.js"; -import { jsonValueSchema } from "../../domain/jsonValue.js"; import { safeParseProcessCapture, type ProcessCapture, @@ -91,7 +90,7 @@ export const createProcessCaptureEvidence = ( predicateType: PROCESS_CAPTURE_EVIDENCE.predicate, operation: PROCESS_CAPTURE_EVIDENCE.operation, parameters: processEvidenceParameters(scenario, capture.manifest.platform), - result: jsonValueSchema.parse(capture), + result: capture, confidence: "observed", authority: "controlled-replay", environment: { diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts index 187f008d1a..f1cc74eecd 100644 --- a/src/application/sqlite/SqliteDatabaseService.ts +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -23,7 +23,6 @@ import { } from "../../domain/sqlite/sqliteDatabase.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { jsonValueSchema } from "../../domain/jsonValue.js"; const OPERATION = "inspect_sqlite_database"; @@ -115,7 +114,7 @@ export class SqliteDatabaseService { createEvidence(execution.subject, execution.provider, { operation: OPERATION, parameters: sqliteInspectionParameters(input.data), - result: jsonValueSchema.parse(report.data), + result: report.data, ...(execution.rawResult === null ? {} : { rawResult: execution.rawResult }), diff --git a/src/contracts/investigationExamples.ts b/src/contracts/investigationExamples.ts index 293bf8d654..3fa2b1a826 100644 --- a/src/contracts/investigationExamples.ts +++ b/src/contracts/investigationExamples.ts @@ -1,5 +1,4 @@ import { createEvidence } from "../domain/evidence.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { FUNCTION_COMPARISON_PROVIDER } from "../domain/comparisonEvidenceIdentity.js"; import { compareProcessCaptureEvidence } from "../domain/process/processComparisonEvidence.js"; import { PROCESS_PROVIDER } from "../domain/process/processEvidenceProvider.js"; @@ -31,7 +30,7 @@ export const FUNCTION_COMPARISON_EVIDENCE = createEvidence( left_evidence_id: left.value.evidence.evidence_id, right_evidence_id: right.value.evidence.evidence_id, }, - result: jsonValueSchema.parse(comparison), + result: comparison, confidence: "derived", authority: "analyst-inference", limitations: comparison.limitations, @@ -48,7 +47,7 @@ const captureEvidence = (scenario: string) => predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: { scenario }, - result: jsonValueSchema.parse(capture), + result: capture, confidence: "observed", authority: "controlled-replay", environment: { diff --git a/src/domain/analysisObservation.ts b/src/domain/analysisObservation.ts index 06ed145a01..f43e8972a8 100644 --- a/src/domain/analysisObservation.ts +++ b/src/domain/analysisObservation.ts @@ -7,7 +7,7 @@ import { type EvidenceSubjectTarget, } from "./evidence.js"; import type { AnalysisProfileCommitment } from "./analysisProfile.js"; -import { jsonObjectSchema, type JsonValue } from "./jsonValue.js"; +import type { JsonValue } from "./jsonValue.js"; import type { RelationshipCoverage } from "./relationshipCoverage.js"; /** Provider observations that can be committed to Evidence. */ @@ -44,7 +44,7 @@ export const createAnalysisExecutionEvidence = ( execution.provider, { operation, - parameters: jsonObjectSchema.parse(input), + parameters: input, result: execution.result, rawResult: execution.rawResult, limitations: execution.limitations, diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts index 2230190472..6d08843610 100644 --- a/src/domain/evidence.test.ts +++ b/src/domain/evidence.test.ts @@ -138,6 +138,102 @@ it("projects known provider fields and preserves open JSON payloads", () => { expect(parseEvidence(evidence)).toEqual(evidence); }); +it("admits typed semantic payloads with omitted optional facts without changing inputs or identity", () => { + interface Selection { + readonly procedure: string; + readonly address?: string; + } + interface SymbolRow { + readonly name: string; + readonly size?: number; + } + interface NativeRecord { + readonly producer: string; + readonly diagnostic?: string; + } + const parameters: Selection = { procedure: "main" }; + const row: SymbolRow = { name: "main" }; + const result = { symbols: [row] }; + const rawResult: NativeRecord = { producer: "authored fixture" }; + const evidence = createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters, + result, + rawResult, + }); + const equivalent = createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters: { procedure: "main" }, + result: { symbols: [{ name: "main" }] }, + rawResult: { producer: "authored fixture" }, + }); + + expect(evidence).toEqual(equivalent); + expect(evidence.evidence_id).toBe(equivalent.evidence_id); + expect(parameters).toEqual({ procedure: "main" }); + expect(result).toEqual({ symbols: [{ name: "main" }] }); + expect(result.symbols[0]).toBe(row); + expect(rawResult).toEqual({ producer: "authored fixture" }); + expect(Object.hasOwn(parameters, "address")).toBe(false); + expect(Object.hasOwn(row, "size")).toBe(false); + expect(Object.hasOwn(rawResult, "diagnostic")).toBe(false); + expect(Object.isFrozen(parameters)).toBe(false); + expect(Object.isFrozen(result)).toBe(false); + expect(Object.isFrozen(row)).toBe(false); + expect(Object.isFrozen(rawResult)).toBe(false); +}); + +it("keeps an omitted, undefined, or null raw payload equivalent", () => { + const observation = { operation: "inspect", parameters: {}, result: true }; + const omitted = createEvidence(TARGET, PROVIDER, observation); + const explicitUndefined = createEvidence(TARGET, PROVIDER, { + ...observation, + rawResult: undefined, + }); + const explicitNull = createEvidence(TARGET, PROVIDER, { + ...observation, + rawResult: null, + }); + + expect(omitted.raw_result).toBeNull(); + expect(explicitUndefined).toEqual(omitted); + expect(explicitNull).toEqual(omitted); +}); + +it.each([ + { field: "parameters", prefix: "parameters" }, + { field: "result", prefix: "normalized_result" }, + { field: "rawResult", prefix: "raw_result" }, +])( + "rejects explicit nested undefined in $field at its Evidence prefix", + ({ field, prefix }) => { + const invalid = { nested: { value: undefined } }; + let failure: unknown; + try { + createEvidence(TARGET, PROVIDER, { + operation: "inspect", + parameters: {}, + result: true, + [field]: invalid, + }); + } catch (cause: unknown) { + failure = cause; + } + + expect(failure).toMatchObject({ + name: "ZodError", + issues: [ + { + path: [prefix], + message: expect.stringContaining("JSON"), + }, + ], + }); + expect(Object.hasOwn(invalid.nested, "value")).toBe(true); + expect(invalid.nested.value).toBeUndefined(); + }, +); + it("preserves identity and bytes when reusing an owned immutable result", () => { const value = { nested: { values: ["observed", 1] } }; const mutable = createEvidence(TARGET, PROVIDER, { diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index bcbc959386..789537f564 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -242,13 +242,14 @@ type ExecutionEnvironment = z.infer; export type EvidenceProvider = Readonly>; -export interface EvidenceObservation { +/** Payloads admitted and snapshotted by the ordinary Evidence constructor. */ +export interface EvidenceObservationInput { readonly predicateType?: string; readonly operation: string; - readonly parameters: Readonly>; - readonly result: JsonValue; + readonly parameters: unknown; + readonly result: unknown; readonly analysisProfile?: AnalysisProfileCommitment; - readonly rawResult?: JsonValue; + readonly rawResult?: unknown; readonly confidence?: "observed" | "derived" | "inferred"; readonly authority?: EvidenceAuthority; readonly environment?: ExecutionEnvironment | null; @@ -260,6 +261,13 @@ export interface EvidenceObservation { readonly evidenceLinks?: readonly string[]; } +/** Producer facts that already promise JSON, including cooperative Evidence input. */ +export interface EvidenceObservation extends EvidenceObservationInput { + readonly parameters: Readonly>; + readonly result: JsonValue; + readonly rawResult?: JsonValue; +} + type EvidenceWithoutId = Omit; /** Content committed by an Evidence ID, excluding local request/display metadata. */ @@ -403,11 +411,11 @@ export interface EvidenceCreationOptions { readonly immutable?: boolean; } -/** Build deterministic Evidence from an immutable artifact subject. */ +/** Admit payloads into deterministic Evidence bound to an immutable artifact subject. */ export const createEvidence = ( target: EvidenceSubjectTarget | BinaryTarget | undefined, provider: EvidenceProvider, - observation: EvidenceObservation, + observation: EvidenceObservationInput, options?: EvidenceCreationOptions, ): Evidence => { const { normalized, sharedResult } = normalizeEvidenceObservation( @@ -460,7 +468,7 @@ export function* createImmutableEvidenceSteps( const normalizeEvidenceObservation = ( target: EvidenceSubjectTarget | BinaryTarget | undefined, provider: EvidenceProvider, - observation: EvidenceObservation, + observation: EvidenceObservationInput, ) => { const subject = target === undefined @@ -509,7 +517,7 @@ const normalizeEvidenceObservation = ( : { relationship_coverage: observation.relationshipCoverage }), locations: [...(observation.locations ?? [])], evidence_links: [...(observation.evidenceLinks ?? [])], - } satisfies JsonValue; + }; const sharedResult = typeof observation.result === "object" && observation.result !== null && diff --git a/src/domain/pcap/pcapEvidence.ts b/src/domain/pcap/pcapEvidence.ts index d2379ac75f..14892f533b 100644 --- a/src/domain/pcap/pcapEvidence.ts +++ b/src/domain/pcap/pcapEvidence.ts @@ -1,5 +1,5 @@ import { createEvidence } from "../evidence.js"; -import { jsonObjectSchema, type JsonValue } from "../jsonValue.js"; +import type { JsonValue } from "../jsonValue.js"; import type { PcapRequest } from "./pcapAnalysis.js"; /** Domain facts needed by both successful and retained partial observations. */ @@ -21,7 +21,7 @@ export const createPcapEvidence = ( ) => createEvidence(observation.subject ?? undefined, observation.provider, { operation: request.operation, - parameters: jsonObjectSchema.parse(request.input), + parameters: request.input, result: observation.result, rawResult: observation.rawResult, limitations: observation.limitations, diff --git a/src/domain/process/processComparisonEvidence.ts b/src/domain/process/processComparisonEvidence.ts index 4a27f80f51..2aa42ba9ee 100644 --- a/src/domain/process/processComparisonEvidence.ts +++ b/src/domain/process/processComparisonEvidence.ts @@ -1,5 +1,4 @@ import { createEvidence, type Evidence } from "../evidence.js"; -import { jsonValueSchema } from "../jsonValue.js"; import type { ProcessCapture } from "./processCaptureParsing.js"; import { compareProcessCaptures, @@ -37,9 +36,9 @@ export const compareProcessCaptureEvidence = ( : { max_capture_age_ms: options.maxCaptureAgeMs }), ...(options.traceSpecification === undefined ? {} - : { trace_spec: jsonValueSchema.parse(options.traceSpecification) }), + : { trace_spec: options.traceSpecification }), }, - result: jsonValueSchema.parse(comparison), + result: comparison, confidence: "derived", authority: "analyst-inference", limitations: comparison.limitations, diff --git a/src/server/registerArtifactComparisonTool.ts b/src/server/registerArtifactComparisonTool.ts index 3e9fe67a40..f21ddb8c7c 100644 --- a/src/server/registerArtifactComparisonTool.ts +++ b/src/server/registerArtifactComparisonTool.ts @@ -6,7 +6,6 @@ import { compareArtifacts } from "../domain/artifactComparison.js"; import { ARTIFACT_COMPARISON_PROVIDER } from "../domain/comparisonEvidenceIdentity.js"; import { createEvidence } from "../domain/evidence.js"; import { readEvidence } from "../domain/evidenceValidation.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { resolveArtifactComparisonEvidence } from "../application/artifacts/ArtifactComparisonEvidence.js"; import type { RecordUnknownInput } from "../domain/residualUnknown.js"; import { recordDerivedEvidence } from "./recordDerivedEvidence.js"; @@ -78,7 +77,7 @@ export const registerArtifactComparisonTool = ( left_evidence_ids: leftEvidenceIds, right_evidence_ids: rightEvidenceIds, }, - result: jsonValueSchema.parse(comparison), + result: comparison, confidence: "derived", authority: "analyst-inference", limitations: comparison.limitations, diff --git a/src/server/registerBundleComparisonTool.ts b/src/server/registerBundleComparisonTool.ts index 1a90b3e839..033f98a938 100644 --- a/src/server/registerBundleComparisonTool.ts +++ b/src/server/registerBundleComparisonTool.ts @@ -5,7 +5,6 @@ import { readImmutableEvidenceBundle } from "../application/EvidenceBundleFiles. import { toolContract } from "../contracts/toolContracts.js"; import { compareBundles } from "../domain/bundleComparison.js"; import { createEvidence } from "../domain/evidence.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; import { BUNDLE_COMPARISON_PROVIDER } from "../domain/investigationProviders.js"; @@ -82,7 +81,7 @@ export const registerBundleComparisonTool = ( right_bundle_sha256: comparison.right_bundle_sha256, record_pairs: input.record_pairs, }, - result: jsonValueSchema.parse(comparison), + result: comparison, confidence: "derived", authority: "analyst-inference", limitations: comparison.limitations, diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts index 4817c183b5..5c036c4f9a 100644 --- a/src/server/registerFunctionComparisonTool.ts +++ b/src/server/registerFunctionComparisonTool.ts @@ -6,7 +6,6 @@ import { toolContract } from "../contracts/toolContracts.js"; import { createEvidence } from "../domain/evidence.js"; import { parseFunctionEvidence } from "../domain/functionDossierEvidence.js"; import { FUNCTION_COMPARISON_PROVIDER } from "../domain/comparisonEvidenceIdentity.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import type { RecordUnknownInput } from "../domain/residualUnknown.js"; import { recordDerivedEvidence } from "./recordDerivedEvidence.js"; import { recordEvidenceSources } from "../application/investigation/recordEvidenceSources.js"; @@ -84,7 +83,7 @@ export const registerFunctionComparisonTool = ( left_evidence_id: leftEvidence.evidence_id, right_evidence_id: rightEvidence.evidence_id, }, - result: jsonValueSchema.parse(comparison), + result: comparison, confidence: "derived", authority: "analyst-inference", limitations: comparison.limitations, diff --git a/src/server/registerInvestigationTools.ts b/src/server/registerInvestigationTools.ts index 689b812e5c..dd211dcbe7 100644 --- a/src/server/registerInvestigationTools.ts +++ b/src/server/registerInvestigationTools.ts @@ -6,7 +6,6 @@ import { toolContract } from "../contracts/toolContracts.js"; import { buildCallPath } from "../domain/callPath.js"; import { findChangedBehavior } from "../domain/changedBehavior.js"; import { createEvidence } from "../domain/evidence.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { correlateStaticAndRuntime } from "../domain/staticRuntimeCorrelation.js"; import { comparisonClosure, @@ -88,7 +87,7 @@ const registerChangedBehavior = ( parameters: { comparison_evidence_ids: [...comparisonEvidenceIds], }, - result: jsonValueSchema.parse(result), + result, confidence: "derived", authority: "analyst-inference", limitations: result.limitations, @@ -161,7 +160,7 @@ const registerCallPath = ( start: input.start.address, goal: input.goal.address, }, - result: jsonValueSchema.parse(result), + result, confidence: "derived", authority: "analyst-inference", limitations: result.limitations, @@ -236,7 +235,7 @@ const registerStaticRuntime = ( parameters: { mapping_count: input.mappings.length, }, - result: jsonValueSchema.parse(result), + result, confidence: "inferred", authority: "analyst-inference", limitations: result.limitations, @@ -311,7 +310,7 @@ const registerReconstruction = ( ({ claim_id: id }) => id, ), }, - result: jsonValueSchema.parse(result), + result, confidence: "derived", authority: "analyst-inference", limitations: result.limitations, From 38719a6fa9a1301eeee647e6749701a9b83f10bb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 24 Oct 2026 14:06:21 -0700 Subject: [PATCH 14/37] feat(environment): explain values an application derives from other fields (#7025) * feat(environment): explain values an application derives from other fields Prompt-built environments withheld creates whose responses carry values the application derives deterministically, which the write model kept as constants or could not match across calls: - A composed string is literal text plus parts read from the fields other rules write: a field's value, its slug, or a clock's calendar digits. A create composes a string only when another entity of the type spells it the same way and none spells it otherwise, and never from two sources serving could tell apart; an update composes an unsent change from members stored as sent and unchanged fields that also spell its value before the call. A composed field no longer pins the members it reads. - A trimmed clock format drops trailing fraction zeros, as Go spells nanoseconds. It is learned from the type's values, a later create or a later update, and spreads to the plan's other clocks unless a value shows a fixed width. - A large integer no entity of the type holds, among distinct ones of the same length and kind, is a generated unique value, served as a seeded integer. Co-Authored-By: Claude Opus 5.5 * docs(environment): record the derived-values verification Gates, tests, write plans over 69 recorded captures and served write tasks on the baseline and on this branch. Co-Authored-By: Claude Opus 5.5 * fix(environment): keep clock spellings, linked ids and word boundaries honest - A later update's clocks take the plan's spelling, sibling-trimmed clocks included, wherever the call's values fit it, so an update whose later call trims only one of two clocks is served. - A held value that ends its fraction in a zero keeps a create's clock at a fixed width, whatever order the held spellings come in. - A large integer that another field of the created entity also holds is linked, as a repeated token is: it is never unique, and the create is withheld as unexplained-change. - Composed parts are delimited by whole characters through the shared joinsBefore/joinsAfter, so an astral letter joins a match to its word. Co-Authored-By: Claude Opus 5.5 * test(environment): keep a same-width fixed clock out of trimming A held .123456780 beside a held .12345678 keeps the create's clock at nine digits in either order; the cases fail on 8ce1127b5. Co-Authored-By: Claude Opus 5.5 * fix(environment): check composed parts and clock fits before they shape a plan - A composed string whose part has nothing to spell refuses the create or update through the apply Result instead of keeping a stale value. - A composition learned on a later call is rebuilt from each earlier call's fields, so an earlier call that left the value otherwise contradicts it. - An update adopts its fitted clock spellings only after the unknown-entity and contradiction checks, so a call on an entity an unmodelled write left unknown no longer trims a served plan's clock. - Sibling trimming on a later create also checks that call's fields, so a sibling clock it shows with a trailing zero stays fixed-width. Co-Authored-By: Claude Opus 5.5 * test(environment): split the derived-values view tests by rule Co-Authored-By: Claude Opus 5.5 * fix(environment): keep fixed-width, zone and generated-integer evidence through a plan - A plan keeps the first value that showed each clock at a fixed width, from held entities and every accepted create or update. No later fit or sibling trim widens that field, and a later call that shows it shorter is withheld. - Two stamped clocks are one composed source only when they share a zone offset. A held entity that holds the target field as anything the parts do not spell, a non-string included, vetoes the composition. The token index is built once, and only when a string is left to compose. - A generated integer that another field repeats as a number at any depth is linked. Compare treats two integers of one JSON kind and length under a unique rule as generated values. Co-Authored-By: Claude Opus 5.5 * test(setup): share registration prewarm host preparation * fix(environment): retain write evidence and match trimmed response clocks * fix(environment): withhold creates whose generated integer another string repeats A unique integer whose digits show in another top-level string that no template or composed rule rebuilds from it (an echoed serialText, or a token like folder-) withholds the create as unexplained-change, since serving would seed the integer and keep the string. A templated copy is still served. Co-Authored-By: Claude Opus 5.5 * fix(environment): hold later creates and compare to the composed and integer forms - A later create whose response spells a composed field otherwise than the plan rebuilds it is withheld as unexplained-change. - A unique field learned from a generated integer accepts only an integer of the same length and kind from a later create, as serving seeds one. - Compare counts a composed field as equal when each side's response spells it from its own fields, which compare under their own rules. Co-Authored-By: Claude Opus 5.5 * fix(environment): preserve composed parts and unique generator families --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/agent-environments.md | 19 +- .../write-derived-values/plan.md | 12 + .../write-derived-values/spec.md | 59 ++ .../write-derived-values/tasks.md | 10 + .../write-derived-values/verification.md | 91 +++ docs/web-network-captures.md | 25 +- .../environmentCompareDiff.test.ts | 352 ++++++++++++ .../environment/environmentCompareDiff.ts | 104 +++- .../environmentStateDerivedAnswer.test.ts | 84 +++ .../environmentStateWriteAnswer.ts | 12 +- .../EnvironmentStateWriteSimulation.ts | 1 + .../environmentStateDerivedWrites.test.ts | 517 ++++++++++++++++++ .../analysisView/environmentStateView.ts | 22 + .../environmentStateWriteApply.test.ts | 78 +++ .../environmentStateWriteApply.ts | 26 +- .../environmentStateWriteCalls.ts | 17 + .../environmentStateWriteChanges.ts | 5 + .../environmentStateWriteClockFormats.test.ts | 111 ++++ .../environmentStateWriteClockFormats.ts | 164 ++++++ .../environmentStateWriteComposed.test.ts | 302 ++++++++++ .../environmentStateWriteComposed.ts | 274 ++++++++++ .../environmentStateWriteCreate.ts | 119 +++- .../environmentStateWriteEffects.ts | 107 +++- .../environmentStateWriteExplain.test.ts | 86 +++ .../environmentStateWriteExplain.ts | 114 +++- .../environmentStateWriteRules.test.ts | 52 ++ .../environmentStateWriteRules.ts | 55 ++ .../environmentStateWriteValues.test.ts | 65 ++- .../environmentStateWriteValues.ts | 95 +++- .../cli/environmentCompareCli.test.ts | 90 +++ .../fixtures/environmentStateDerivedValues.ts | 484 ++++++++++++++++ 31 files changed, 3467 insertions(+), 85 deletions(-) create mode 100644 docs/specs/agent-environments/write-derived-values/plan.md create mode 100644 docs/specs/agent-environments/write-derived-values/spec.md create mode 100644 docs/specs/agent-environments/write-derived-values/tasks.md create mode 100644 docs/specs/agent-environments/write-derived-values/verification.md create mode 100644 src/application/environment/environmentStateDerivedAnswer.test.ts create mode 100644 src/domain/analysisView/environmentStateDerivedWrites.test.ts create mode 100644 src/domain/analysisView/environmentStateWriteClockFormats.test.ts create mode 100644 src/domain/analysisView/environmentStateWriteClockFormats.ts create mode 100644 src/domain/analysisView/environmentStateWriteComposed.test.ts create mode 100644 src/domain/analysisView/environmentStateWriteComposed.ts create mode 100644 src/domain/analysisView/environmentStateWriteRules.test.ts create mode 100644 tests/fixtures/environmentStateDerivedValues.ts diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 62db30bb65..4b812f9659 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -721,7 +721,7 @@ every MCP session and `data/get` then see: | Effect | Call | | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. | -| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, unique and renewed changes the plan models. | +| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, unique and renewed changes the plan models. | | `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), with any sent members; it is served only when a recorded call found those fields already set, so a toggle never is. | | `delete` | Removes the entity the path names. | | bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). | @@ -923,6 +923,9 @@ Known limits of served writes: updates require `updated_at` to match the entity (a `held-clock`). - A slug derived from a title can repeat across served creates with the same title, where the application would make it unique. +- A `composed` string copies sent text as sent: an application that escapes or + renders it (HTML entities, Markdown) differs for text that needs it, as + BookStack's `description_html` would for a description holding `&`. - A served create's sequence id is the state's next value, not the gaps an application's own sequence would leave. - Nested clocks are served in the spelling of the write's own response, which @@ -1057,9 +1060,11 @@ Numeric arrays retain that containing field's clock context through nested arrays; an object's own fields establish their own context. Numeric values at the body root, in root arrays, or under measurement fields remain literal. -Generated strings normalize only where this confirmed, served write's entity -response rules identify generation. A `unique` rule or UUID identity rule must -still compare two UUIDs; a `renewed` rule applies only at its recorded pointers, +Generated values normalize only where this confirmed, served write's entity +response rules identify generation. A UUID identity rule must still compare two +UUIDs, and a `unique` rule two UUIDs or two generated integers (10 to 20 +digits) of one JSON kind and length; a `renewed` rule applies only at its +recorded pointers, with `*` matching actual array items. Token/sequence strings retain the existing format check: equal length of at least eight, the same character classes, and a digit. Numeric sequences continue to compare literally unless an ID mapping @@ -1071,6 +1076,12 @@ Admitted generated values must also preserve aliases and distinct identities within that response. UUID identity uses the generator's case-insensitive matching; token and sequence strings retain their literal identity. This correspondence ends with the answer and is not inferred across calls. +A `composed` response field must spell the value its parts build from that +response's own fields. Its parts then compare by their rendered text or their +source fields' clock and ID rules. A coherent archive URL can therefore differ +only in normalized clock digits without reporting a value difference; an +incoherent URL still differs even when both sides repeat the same stale text. +Normalized compositions count under `generated`; source fields are counted once. The report counts each kind of normalised value; the generation rules remain model-derived facts, rather than new observations of the application. diff --git a/docs/specs/agent-environments/write-derived-values/plan.md b/docs/specs/agent-environments/write-derived-values/plan.md new file mode 100644 index 0000000000..a5ae3f29f8 --- /dev/null +++ b/docs/specs/agent-environments/write-derived-values/plan.md @@ -0,0 +1,12 @@ +# Implementation plan + +1. **Clock formats.** A `trimmed` ISO format, `fixedWidth` on a clock reading, fitting a format to another value, and + calendar digits, in a clock formats module. +2. **Composed strings.** A module that indexes a create's candidate parts once, segments a string, checks the type's + other entities and an update's before value, and spells the string from written fields. +3. **Explaining.** Create explanation composes unexplained strings after templates, learns trimmed clocks from the + type and generated integers; update explanation composes unsent changes after the existing rules. +4. **Later calls.** A later create or update widens the plan's clocks to a trimmed spelling it shows; a later create's + unique value may be a generated integer. +5. **Serving.** Derived fields are rebuilt after other rules; unique integers are seeded by length and kind. +6. **Schema, tests and docs.** The plan schema, view, serving and unit tests, and the write-plan docs. diff --git a/docs/specs/agent-environments/write-derived-values/spec.md b/docs/specs/agent-environments/write-derived-values/spec.md new file mode 100644 index 0000000000..efc90d707a --- /dev/null +++ b/docs/specs/agent-environments/write-derived-values/spec.md @@ -0,0 +1,59 @@ +# Explain values an application derives from other fields + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +Environments an agent built from a prompt withheld creates in three of five applications, each for a value the write +model could not explain, though the application derives it deterministically: + +- Linkding's `web_archive_snapshot_url` is `https://web.archive.org/web//`. + The plan kept it as a constant, which pinned `url` to its recorded value, so a later create with another URL was + refused. +- Grafana's folder `url` is `/dashboards/f//`, which pinned `uid` the same way, and its numeric + `id` is a large time-based integer the plan copied from the first create, which the second create's read-back + contradicted. +- Homebox, written in Go, spells nanosecond clocks with trailing fraction zeros dropped (`…34.38812218Z`), so a later + create's clock did not follow the format learned from the first. + +## Behaviour + +- **Composed strings.** A created field no other rule explains may be a `composed` string: literal text and parts read + from the fields other rules write, each a field's value, its slug, or a clock's calendar digits (`YYYYMMDDhhmmss`, + `YYYYMMDD`, `YYYY-MM-DD`, `YYYY/MM/DD`, in the clock's zone). Parts are found left to right, longest first, delimited + by non-alphanumerics; generated values may be short, other values need eight characters, as a copy does. A create + composes a string only when at least one other entity of the type spells it with the same parts and none spells it + otherwise, and never when one text comes from two sources serving could tell apart (two clocks the write stamps are + one source when they spell the same zone), and never when another entity holds the field as something other than the + parts spell, a non-string value included. An update's unsent change may be composed from members it stored as sent and + fields it left as they were, when the same parts also spell the value before the call and no other entity spells it + otherwise. Serving rebuilds the string from the written fields after every other rule and refuses a call when a part's + field holds nothing to spell, and a composed field no longer pins the members it reads. A composition learned on a + later call is checked against every earlier call, and a later create whose response spells a composed field otherwise + is withheld. +- **Trimmed clocks.** A clock format may be `trimmed`: its values differ only in how many fraction digits they show and + none ends a fraction in a zero. It is learned from the type's held values of the field, from a later create, and from + a later update; other clocks the plan stamps take the spelling too, and no clock takes it once a held entity, the + exemplar or any call the plan accepted showed the field at a fixed width; a later call that then shows it shorter is + withheld. `fraction` is the most digits shown, and serving drops trailing zeros and the point with them. +- **Generated integers.** A created integer of 10 to 20 digits that no entity of the type holds, where every held entity + holds a distinct one of the same length and JSON kind and none is a clock, is `unique`. Serving draws a seeded integer + of that length and kind that no held entity has. One that another field of the created entity also holds, as a value, + a number at any depth or a delimited part of a nested string, is not `unique`, and the create is withheld as + `unexplained-change`, since seeding cannot keep the two equal. A create is also withheld when another string holds its + digits and no `template` or `composed` rule rebuilds that string from it, or when a later create answers the field + with anything but an integer of the same length and kind. Compare treats two such integers of one JSON kind and length + under a `unique` rule as generated values. + +Plans that need none of this are unchanged. + +Later creates must retain the exemplar's unique-value family (UUID, integer with +the same digit count and JSON kind, or opaque token). Their recorded composed +fields must equal the values the plan rebuilds; simulation cannot replace a +contradictory captured value and then count that replacement as confirmation. + +## Acceptance + +- The prompt-built Linkding, Homebox and Grafana captures serve the creates and renames they withheld, and write + tasks that use them pass. +- No write tool of the 50 published captures or the earlier prompt-built captures goes from served to withheld. diff --git a/docs/specs/agent-environments/write-derived-values/tasks.md b/docs/specs/agent-environments/write-derived-values/tasks.md new file mode 100644 index 0000000000..ae9e074b47 --- /dev/null +++ b/docs/specs/agent-environments/write-derived-values/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] Read and fit trimmed clock formats, and spell calendar digits. +- [x] Compose created strings and unsent update changes from other fields. +- [x] Learn generated integers as unique values and serve them by length and kind. +- [x] Widen plan clocks on later creates and updates, and spread a trimmed spelling to the plan's other clocks. +- [x] Add view, serving and unit tests. +- [x] Document the rules. +- [x] Measure the prompt-built captures, the 50 published captures and the served write tasks. +- [x] Run static, formatting, documentation and test gates. diff --git a/docs/specs/agent-environments/write-derived-values/verification.md b/docs/specs/agent-environments/write-derived-values/verification.md new file mode 100644 index 0000000000..5f74a2d775 --- /dev/null +++ b/docs/specs/agent-environments/write-derived-values/verification.md @@ -0,0 +1,91 @@ +# Verification + +Executed on macOS with Node v24.22.1 at `a1edeafd` (main `79845063` plus this branch). The prompt-built recordings and +evaluations used a local, unpushed build of this branch merged with the scenario-timing branch (#6967), whose request +times the clock rules need from browser scenarios, on Ubuntu 24.04 VMs with snap Chromium 155. The baseline is the same +merge without this branch. + +## Gates + +Follow-up on 2026-10-25, after merging main `a5bfd0440`, with Node 24.22.1 and npm 11.16.0 on macOS: + +- Environment write/compare source tests: 26 files, 839 passed. +- Analysis-view CLI/MCP, environment served-write MCP, environment-compare CLI, managed bundle read guards, and setup + registration prewarm boundaries: 6 distinct files, 77 passed. The managed low-headroom failure is fixed by main's + Evidence ownership change (#7036); the setup tests share host preparation to satisfy lint without removing cases. +- `check:fast`, `format:check`, and `docs:check`: passed. +- New capture-to-plan regressions reproduce missing composition parts on earlier updates (null and structured sources) + and fixed-width clock evidence from either the selected entity or another held entity. Rule tests exercise several + separator/zone groups in both orders, choosing each group's widest trimmed peer while preserving fixed-width fields. +- Compare uses the domain clock matcher on parsed readings. Tests cover shorter fractions, absent fractions, maximum + width, trailing zeros, separators, zones, and response-entity scope; the compiled CLI accepts a Go-style create whose + live clocks have eight digits and served clocks have none, preserving all input Evidence files. +- The original application's recordings and model evaluations below were not rerun in this follow-up. + +Original author validation: + +- `npm run -s typecheck`, `lint`, `format:check`, `docs:check`, `mcp-catalog:check`, `verify:test-discovery`, + `scan:todos` and `npx knip`: passed. +- `npx vitest run --project domain --project services --project adapters`: 17,246 passed, 25 skipped, 1 failed ("stops + Inspector target authorization after the current stat settles"), which fails the same way on main in this environment. +- `npm run test:prepare`, then `npm run test:focused --` the environment build, export and serve CLI boundaries, the + analysis-view CLI and MCP boundaries, the environment serve, served-writes and bundle MCP boundaries, MCP schema + validation and output schema constraints: 10 files, 82 passed. + +## Tests + +- View: Go-spelled tag creates and renames are served with a trimmed clock; bookmark creates are served with a composed + snapshot address and accept any URL; folder creates and renames are served with a composed address and a generated + integer id. +- Serving: a served tag create spells its clocks without trailing zeros; a served bookmark create composes its snapshot + address from the served clock and the sent URL; a served folder create gets a fresh sixteen-digit id and an address + that follows its uid and title, and a rename moves the address. +- Unit: fitting clock formats (trimmed, fixed width, separator and zone), calendar digits, seeded integers, generated + integer detection, composition and its refusals (one call alone, a held entity spelling otherwise, two sources with + one text, a member stored otherwise than sent, a value the parts do not spell before the call), and a wide body + composed within the existing time limit. + +## Write plans over recorded captures + +Every write plan of 69 captures, built by the baseline and by this branch: the 50 published captures, 18 captures +Claude Code recorded from a prompt for the prompt branch's checks, and one new timed Grafana recording. + +| Captures | Write plans | Served on the baseline | Served on this branch | +| ------------------------- | ----------- | ---------------------- | --------------------- | +| 50 published | 187 | 68 | 68 | +| 19 recorded from a prompt | 120 | 56 | 61 | + +- No write goes from served to withheld. The five gains are Linkding's bookmark create, Homebox's tag create, and + Grafana's folder create in two recordings and folder rename in the timed one. +- Published plans change only where a value is now composed: Linkding's bookmark create and BookStack's book create and + update (`description_html` as `

` around the description) now accept the member they read. +- Unchanged by design: Grafana's published folder, whose uid and title slug are the same text, so its address names two + sources; Ghost's post update, whose `lexical` keeps the `derived` rule that explained it before; BookStack's page HTML + and Jellyfin's playlist path, which other pages and playlists of the capture spell otherwise. +- What still withholds a create the composed value now explains: a later Linkding create that sends a member the first + did not (`shared`), Homebox item creates that differ from the first location create, and a Linkding create whose + read-back lists `tag_names` sorted. + +## Served write tasks + +The same timed captures, rebuilt by the baseline and by this branch, with the models `fireworks_ai/gpt-oss-120b` and +`fireworks_ai/glm-5p3-flash`. Create tasks are new; recorded write tasks are the applications' existing ones with entity +counts taken relative to each bundle's state. + +| Application | Tasks | Baseline (gpt-oss, GLM) | This branch (gpt-oss, GLM) | +| ----------- | ---------------------------------- | ----------------------- | -------------------------- | +| Linkding | 2 bookmark creates | 0/2, 0/2 | 2/2, 2/2 | +| Linkding | 3 recorded (update, notes, delete) | 3/3, 3/3 | 3/3, 3/3 | +| Homebox | 2 tag creates | 0/2, 0/2 | 2/2, 2/2 | +| Homebox | 2 recorded deletes | 0/2, 0/2 | 0/2, 0/2 | +| Grafana | folder create and rename | not served | 1/2, 2/2 | +| Grafana | 2 recorded deletes | | 2/2, 2/2 | + +- Smoke and retrieval match on both builds (Linkding 9/9 and 8/9, Homebox 3/3 and 3/3); Grafana's new recording passes + its smoke task and 9/9 retrieval with both models. +- gpt-oss missed the Grafana rename by sending the update without the title it was asked to change, ten times; the + served environment refused each call with the members it accepts. +- Homebox's recorded deletes fail on both builds: this recording's served entity list holds its locations only, so the + models never find the item to delete. +- The created bookmarks' snapshot addresses and the created and renamed folders' addresses are checked from the + environment's state. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 866494acb7..7459535193 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -740,9 +740,22 @@ explains it: generated `identity` (a sequence, UUID or token, by the recorded value's shape), `echo` of a body member, `copy` of a body string of 8 or more characters, a `path` value, a `slug` of a body member, a `clock` value within 10 minutes of the request in its recorded format, a `sequence` one past the -largest value of its collection or type, a `unique` UUID or opaque token, a -`nested-clock` object whose clocks the request stamped, or a `template` around -a generated value; other fields keep the observed value. A body that wraps one +largest value of its collection or type, a `unique` UUID, opaque token or +integer of 10 or more digits that no entity of the type holds (where every one +holds a distinct integer of the same length, as time-based ids do), a +`nested-clock` object whose clocks the request stamped, a `template` around +a generated value, or a `composed` string; other fields keep the observed value. +A `composed` string is literal text and parts read from the fields other rules +write: a field's value, its slug, or a clock's calendar digits +(`YYYYMMDDhhmmss`, `YYYYMMDD`, `YYYY-MM-DD` or `YYYY/MM/DD`, in the clock's +zone). A create composes a string only when at least one other entity of the +type spells it with the same parts and none spells it otherwise, and never when +one text could come from two sources serving could tell apart. A clock format +is `trimmed` when the field's values differ only in how many fraction digits +they show and none ends a fraction in a zero, as Go spells nanoseconds: its +`fraction` is the most it shows, and serving drops trailing zeros and the point +with them. Other clocks the plan stamps take that spelling too unless their own +value shows a fixed width. A body that wraps one object in a one-element array is that object, wrapped so on every call (`body.envelope`). An update's members echo when stored as sent; a member stored differently is accepted only with its @@ -750,8 +763,10 @@ recorded value (`body.fixed`) and stores the value the call that showed it recorded (the `value` of its `constant` rule), and a field it changed without sending must be a clock, a nested clock, an increment, a slug, a fresh `unique` value, a `stamp` of the acting user, a value `derived` from the one member it -stores nowhere, or for an item action a `set` of a scalar to the value it -showed, or the plan is `unexplained-change`; an increment, slug, stamp, derived +stores nowhere, a `composed` string whose parts read members it stored as sent +or fields it left as they were and spell the value before the call too, or for +an item action a `set` of a scalar to the value it showed, or the plan is +`unexplained-change`; an increment, slug, stamp, derived value or set must hold for every call of the tool, and an action is served only once a call found what it sets already set. A write is served only with the query its first call sent, which serving applies to a call that diff --git a/src/application/environment/environmentCompareDiff.test.ts b/src/application/environment/environmentCompareDiff.test.ts index 53c52e2208..174cf341b4 100644 --- a/src/application/environment/environmentCompareDiff.test.ts +++ b/src/application/environment/environmentCompareDiff.test.ts @@ -4,6 +4,7 @@ import type { JsonValue } from "../../domain/jsonValue.js"; import type { FieldRule } from "../../domain/analysisView/environmentStateWriteRules.js"; import { compareAnswers } from "./environmentCompareDiff.js"; import { IdMap, learnIds } from "./environmentCompareIds.js"; +import { bookmark } from "../../../tests/fixtures/environmentStateDerivedValues.js"; const compare = ( live: unknown, @@ -34,6 +35,233 @@ describe("early-year clock formats", () => { ); }); +describe("composed response values", () => { + const rules: readonly FieldRule[] = [ + { + field: "date_added", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 0, zone: "Z" }, + }, + { + field: "date_modified", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 0, zone: "Z" }, + }, + { + field: "web_archive_snapshot_url", + rule: "composed", + parts: [ + { text: "https://web.archive.org/web/" }, + { from: "date_added", as: "clock", pattern: "YYYYMMDDhhmmss" }, + { text: "/" }, + { from: "url", as: "value" }, + ], + }, + ]; + const live = bookmark( + 1, + "https://example.test/page", + "A bookmark", + "2026-10-20T09:00:00Z", + ); + const env = bookmark( + 1, + "https://example.test/page", + "A bookmark", + "2026-10-17T18:00:00Z", + ); + + it("compares coherent archive URLs through their clock parts without counting source fields twice", () => { + const result = compare(live, env, new IdMap(), [201, 201], rules); + expect(result.differences).toEqual([]); + expect(result.normalised).toMatchObject({ clocks: 2, generated: 1 }); + }); + + it("compares a composed ID through the source field's mapping even when the address comes first", () => { + const map = new IdMap(); + map.learn(101, 1); + const result = compare( + { url: "/folders/101", id: 101 }, + { url: "/folders/1", id: 1 }, + map, + [201, 201], + [ + { + field: "url", + rule: "composed", + parts: [{ text: "/folders/" }, { from: "id", as: "value" }], + }, + ], + ); + expect(result.differences).toEqual([]); + expect(result.normalised).toMatchObject({ ids: 1, generated: 1 }); + }); + + it.each(["live", "env", "both"] as const)( + "reports an incoherent %s composition, including equal but stale URLs", + (side) => { + const invalid = "https://web.archive.org/web/stale"; + const result = compare( + side === "env" ? live : { ...live, web_archive_snapshot_url: invalid }, + side === "live" ? env : { ...env, web_archive_snapshot_url: invalid }, + new IdMap(), + [201, 201], + rules, + ); + expect(result.differences).toMatchObject([ + { category: "value", pointer: "/web_archive_snapshot_url" }, + ]); + expect(result.normalised.generated).toBe(0); + }, + ); + + it("preserves actual input differences although each address correctly spells its own response", () => { + const different = bookmark( + 1, + "https://example.test/other", + "A bookmark", + "2026-10-17T18:00:00Z", + ); + const result = compare(live, different, new IdMap(), [201, 201], rules); + expect(result.differences.map(({ pointer }) => pointer).sort()).toEqual([ + "/url", + "/web_archive_snapshot_url", + ]); + expect(result.normalised.generated).toBe(0); + }); + + it("does not normalize a source's unproven generated-looking business value", () => { + const result = compare( + { url: "/folders/token00001", token: "token00001" }, + { url: "/folders/token00002", token: "token00002" }, + new IdMap(), + [201, 201], + [ + { + field: "url", + rule: "composed", + parts: [{ text: "/folders/" }, { from: "token", as: "value" }], + }, + ], + ); + expect(result.differences.map(({ pointer }) => pointer).sort()).toEqual([ + "/token", + "/url", + ]); + }); + + it("compares the projected slug while retaining a difference in its source title", () => { + const result = compare( + { + title: "Hello!", + date_added: "2026-10-20T09:00:00Z", + url: "/snapshot/20261020/hello", + }, + { + title: "Hello", + date_added: "2026-10-17T18:00:00Z", + url: "/snapshot/20261017/hello", + }, + new IdMap(), + [201, 201], + [ + rules[0]!, + { + field: "url", + rule: "composed", + parts: [ + { text: "/snapshot/" }, + { from: "date_added", as: "clock", pattern: "YYYYMMDD" }, + { text: "/" }, + { from: "title", as: "slug" }, + ], + }, + ], + ); + expect(result.differences).toMatchObject([ + { category: "value", pointer: "/title" }, + ]); + expect(result.normalised).toMatchObject({ clocks: 1, generated: 1 }); + }); +}); + +describe("trimmed response clocks", () => { + const rules: readonly FieldRule[] = [ + { + field: "updatedAt", + rule: "clock", + format: { + kind: "iso", + separator: "T", + zone: "Z", + fraction: 8, + trimmed: true, + }, + }, + ]; + + it.each([".12345678", ".123", ""])( + "normalizes a learned trimmed width %s against the served millisecond width", + (fraction) => { + const result = compare( + { updatedAt: `2026-03-04T05:06:07${fraction}Z` }, + { updatedAt: "2026-03-04T05:06:08.321Z" }, + new IdMap(), + [200, 200], + rules, + ); + expect(result.differences).toEqual([]); + expect(result.normalised.clocks).toBe(1); + }, + ); + + it.each([ + ["2026-03-04T05:06:07.123456789Z", "2026-03-04T05:06:08.321Z"], + ["2026-03-04T05:06:07.123456780Z", "2026-03-04T05:06:08.321Z"], + ["2026-03-04T05:06:07.120Z", "2026-03-04T05:06:08.320Z"], + ["2026-03-04T05:06:07.123+01:00", "2026-03-04T05:06:08.321Z"], + ["2026-03-04 05:06:07.123Z", "2026-03-04T05:06:08.321Z"], + ])( + "reports a pair outside the learned trimmed format (%s, %s)", + (live, env) => { + const result = compare( + { updatedAt: live }, + { updatedAt: env }, + new IdMap(), + [200, 200], + rules, + ); + expect(result.differences).toMatchObject([ + { category: "clock-format", pointer: "/updatedAt" }, + ]); + expect(result.normalised.clocks).toBe(0); + }, + ); + + it("keeps trimmed normalization within the confirmed response entity", () => { + const answer = (fraction: string) => ({ + status: 200, + body: { + kind: "json" as const, + value: { + data: { updatedAt: `2026-03-04T05:06:07.${fraction}Z` }, + updatedAt: `2026-03-04T05:06:07.${fraction}Z`, + }, + }, + }); + const result = compareAnswers( + answer("12345678"), + answer("123"), + new IdMap(), + { at: "/data", rules }, + ); + expect(result.differences).toMatchObject([ + { category: "clock-format", pointer: "/updatedAt" }, + ]); + expect(result.normalised.clocks).toBe(1); + }); +}); + const registerGeneratedValueComparisonCases = () => { it.each([ { @@ -106,6 +334,57 @@ const registerGeneratedValueComparisonCases = () => { }); }; +describe("composed response fields", () => { + const rules: readonly FieldRule[] = [ + { + field: "date_added", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 0, zone: "Z" }, + }, + { field: "url", rule: "echo" }, + { + field: "web_archive_snapshot_url", + rule: "composed", + parts: [ + { text: "https://web.archive.org/web/" }, + { from: "date_added", as: "clock", pattern: "YYYYMMDDhhmmss" }, + { text: "/" }, + { from: "url", as: "value" }, + ], + }, + ]; + const bookmark = (stamp: string, snapshot: string) => ({ + date_added: `2026-10-01T09:00:${stamp}Z`, + url: "https://quarry.example.test/ledger", + web_archive_snapshot_url: `https://web.archive.org/web/${snapshot}/https://quarry.example.test/ledger`, + }); + + it("normalizes a composed field each side spells from its own fields", () => { + const result = compare( + bookmark("00", "20261001090000"), + bookmark("05", "20261001090005"), + new IdMap(), + [200, 200], + rules, + ); + expect(result.differences).toEqual([]); + expect(result.normalised).toMatchObject({ clocks: 1, generated: 1 }); + }); + + it("reports a composed field one side does not spell from its own fields", () => { + const result = compare( + bookmark("00", "20261001090000"), + bookmark("05", "20261001090009"), + new IdMap(), + [200, 200], + rules, + ); + expect( + result.differences.map(({ category, pointer }) => [category, pointer]), + ).toEqual([["value", "/web_archive_snapshot_url"]]); + }); +}); + describe("compareAnswers", () => { registerGeneratedValueComparisonCases(); @@ -671,3 +950,76 @@ describe("raw ids a mapping gives to other entities", () => { ).toEqual([]); }); }); + +describe("generated integer identities", () => { + const unique: readonly FieldRule[] = [{ field: "id", rule: "unique" }]; + + it.each([ + { + name: "numbers of one length", + live: 1_729_791_234_567_890, + env: 1_000_000_000_123_456, + rules: unique, + generated: 1, + }, + { + name: "digit strings of one length", + live: "1729791234567890", + env: "1000000000123456", + rules: unique, + generated: 1, + }, + { + name: "numbers of different lengths", + live: 1_729_791_234_567_890, + env: 100_000_000_012_345, + rules: unique, + generated: 0, + }, + { + name: "a number and a digit string", + live: 1_729_791_234_567_890, + env: "1000000000123456", + rules: unique, + generated: 0, + }, + { + name: "short integers", + live: 17, + env: 18, + rules: unique, + generated: 0, + }, + { + name: "integers no rule generates", + live: 1_729_791_234_567_890, + env: 1_000_000_000_123_456, + rules: [], + generated: 0, + }, + ])( + "matches $name only when a unique rule generates both", + ({ live, env, rules, generated }) => { + const result = compare( + { id: live }, + { id: env }, + new IdMap(), + [200, 200], + rules, + ); + expect(result.normalised.generated).toBe(generated); + expect(result.differences).toEqual( + generated === 1 + ? [] + : [ + { + category: "value", + pointer: "/id", + live: JSON.stringify(live), + env: JSON.stringify(env), + }, + ], + ); + }, + ); +}); diff --git a/src/application/environment/environmentCompareDiff.ts b/src/application/environment/environmentCompareDiff.ts index b028e99a12..25d2db8c13 100644 --- a/src/application/environment/environmentCompareDiff.ts +++ b/src/application/environment/environmentCompareDiff.ts @@ -1,9 +1,16 @@ import type { JsonObject } from "../../domain/analysisView/environmentStateRules.js"; import type { FieldRule } from "../../domain/analysisView/environmentStateWriteRules.js"; +import { matchesClockFormat } from "../../domain/analysisView/environmentStateWriteClockFormats.js"; import { + composedPartText, + composedValue, +} from "../../domain/analysisView/environmentStateWriteComposed.js"; +import { + likeGeneratedInteger, isUuid, readClock, type ClockFormat, + type ClockReading, } from "../../domain/analysisView/environmentStateWriteValues.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import type { ReplayedBody } from "./environmentSource.js"; @@ -76,18 +83,17 @@ const formatKey = (format: ClockFormat): string => : `iso${format.separator}${String(format.fraction)}${format.zone === "" ? "" : format.zone === "Z" ? "Z" : "offset"}`; /** ISO text has its own clock syntax; numeric epochs require a time-named field. */ -const clockFormat = ( +const clockReading = ( value: JsonValue, key: string | undefined, -): string | undefined => { +): ClockReading | undefined => { if ( typeof value === "number" && (key === undefined || (!TIME_FIELD.test(key) && !TIME_FIELD_CAMEL.test(key))) ) return undefined; - const clock = readClock(value); - return clock === undefined ? undefined : formatKey(clock.format); + return readClock(value); }; const characterClasses = (text: string): string => @@ -108,14 +114,21 @@ interface Generation { readonly pointer: string; } -/** Whether a confirmed generation rule applies here and both strings retain its generated format. */ +/** Whether a confirmed generation rule applies here and both values retain its generated format. */ const generatedPair = ( - live: string, - env: string, + live: string | number, + env: string | number, generation: Generation | undefined, ): boolean => { if (generation === undefined) return false; const { rule, pointer } = generation; + if ( + rule.rule === "unique" && + pointer === "" && + likeGeneratedInteger(live, env) + ) + return true; + if (typeof live !== "string" || typeof env !== "string") return false; if (rule.rule === "renewed") return rule.pointers.includes(pointer) && isUuid(live) && isUuid(env); if (pointer !== "") return false; @@ -136,6 +149,7 @@ const generatedPair = ( class Differ { readonly #rules: ReadonlyMap; + readonly #composedRules: readonly Extract[]; readonly #generatedToEnv = new Map(); readonly #generatedToLive = new Map(); readonly differences: Difference[] = []; @@ -159,6 +173,9 @@ class Differ { }, ) { this.#rules = new Map(entity.rules.map((rule) => [rule.field, rule])); + this.#composedRules = entity.rules.filter( + (rule) => rule.rule === "composed", + ); } add(difference: Difference): void { @@ -166,13 +183,21 @@ class Differ { } /** One answer's generated identities must rename consistently in both directions. */ - #pairGenerated(live: string, env: string, rule: FieldRule): boolean { + #pairGenerated( + live: string | number, + env: string | number, + rule: FieldRule, + ): boolean { const uuid = rule.rule === "unique" || rule.rule === "renewed" || (rule.rule === "identity" && rule.generator === "uuid"); - const identity = (value: string) => - uuid ? `uuid:${value.toLowerCase()}` : `string:${value}`; + const identity = (value: string | number) => + typeof value === "number" + ? `number:${String(value)}` + : uuid + ? `uuid:${value.toLowerCase()}` + : `string:${value}`; const liveKey = identity(live); const envKey = identity(env); const knownEnv = this.#generatedToEnv.get(liveKey); @@ -222,12 +247,12 @@ class Differ { generation?: Generation, ): void { const key = at.clockKey ?? at.key; - const liveClock = clockFormat(live, key); - const envClock = clockFormat(env, key); + const liveClock = clockReading(live, key); + const envClock = clockReading(env, key); const generatedMatch = generation !== undefined && - typeof live === "string" && - typeof env === "string" && + (typeof live === "string" || typeof live === "number") && + (typeof env === "string" || typeof env === "number") && liveClock === undefined && envClock === undefined && !this.map.hasLive(live) && @@ -267,7 +292,17 @@ class Differ { return; } if (liveClock !== undefined && envClock !== undefined) { - if (liveClock === envClock) this.normalised.clocks += 1; + const rule = generation?.pointer === "" ? generation.rule : undefined; + const format = + rule?.rule === "clock" || rule?.rule === "held-clock" + ? rule.format + : undefined; + const matches = + format?.kind === "iso" && format.trimmed === true + ? matchesClockFormat(format, liveClock) && + matchesClockFormat(format, envClock) + : formatKey(liveClock.format) === formatKey(envClock.format); + if (matches) this.normalised.clocks += 1; else this.add({ category: "clock-format", @@ -387,6 +422,10 @@ class Differ { : key; }; const matched = new Set(); + const composed = + self && this.#composedRules.length > 0 + ? { agreed: new Set(), fields: new Map() } + : undefined; for (const [key, value] of Object.entries(live)) { const child = `${pointer}/${escapePointer(key)}`; const other = envKey(key); @@ -399,6 +438,11 @@ class Differ { else { matched.add(other); const rule = self ? this.#rules.get(key) : undefined; + if (rule?.rule === "composed" && composed !== undefined) { + composed.fields.set(key, other); + continue; + } + const before = this.differences.length; this.value( value, env[other]!, @@ -417,6 +461,36 @@ class Differ { pointer: `${generation.pointer}/${escapePointer(key)}`, }, ); + if (this.differences.length === before) composed?.agreed.add(key); + } + } + // Compare source fields once, then reuse their clock/identity decisions. + // Validate each response's spelling before normalizing its composition. + if (composed !== undefined) { + for (const rule of this.#composedRules) { + const other = composed.fields.get(rule.field); + if (other === undefined) continue; + const coherent = + composedValue(rule.parts, live) === live[rule.field] && + composedValue(rule.parts, env) === env[other]; + if ( + coherent && + rule.parts.every( + (part) => + !("from" in part) || + composed.agreed.has(part.from) || + composedPartText(part, live) === composedPartText(part, env), + ) + ) { + composed.agreed.add(rule.field); + if (live[rule.field] !== env[other]) this.normalised.generated += 1; + } else + this.add({ + category: "value", + pointer: `${pointer}/${escapePointer(rule.field)}`, + live: shown(live[rule.field]), + env: shown(env[other]), + }); } } for (const [key, value] of Object.entries(env)) diff --git a/src/application/environment/environmentStateDerivedAnswer.test.ts b/src/application/environment/environmentStateDerivedAnswer.test.ts new file mode 100644 index 0000000000..9ec2dca6c8 --- /dev/null +++ b/src/application/environment/environmentStateDerivedAnswer.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; + +import { + BOOKMARK_ROWS, + FOLDER_ROWS, + TAG_ROWS, +} from "../../../tests/fixtures/environmentStateDerivedValues.js"; +import { servedFrom } from "../../../tests/fixtures/environmentStateServing.js"; +import { + bookClock, + numberedUuid, +} from "../../../tests/fixtures/environmentStateWrites.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; + +/** The clock the first served write of a capture of `rows` stamps: one second past the served clock's start. */ +const firstServedClock = (rows: readonly unknown[]): string => + bookClock(rows.length + 1); + +const objectOf = (body: JsonValue | undefined): Record => { + if (typeof body !== "object" || body === null || Array.isArray(body)) + throw new Error("Expected an object body"); + return body; +}; + +describe("served values an application derives from others", () => { + it("stamps a created tag's clocks with trailing fraction zeros dropped, as the capture spelled them", () => { + const tags = servedFrom(TAG_ROWS); + const created = objectOf( + tags.answer("post_tags", { body: { name: "Spare" } }).body, + ); + expect(created).toMatchObject({ + name: "Spare", + createdAt: firstServedClock(TAG_ROWS), + updatedAt: firstServedClock(TAG_ROWS), + }); + }); + + it("composes a created bookmark's snapshot address from the served clock and the sent URL", () => { + const bookmarks = servedFrom(BOOKMARK_ROWS); + const url = "https://pier.example.test/inspection"; + const created = objectOf( + bookmarks.answer("post_bookmarks", { body: { url, title: "Pier" } }).body, + ); + const digits = firstServedClock(BOOKMARK_ROWS) + .replace(/\D/gu, "") + .slice(0, 14); + expect(created).toMatchObject({ + url, + web_archive_snapshot_url: `https://web.archive.org/web/${digits}/${url}`, + }); + }); + + it("gives a created folder a fresh numeric id and an address that follows its uid and title, and renames it", () => { + const folders = servedFrom(FOLDER_ROWS); + const uid = numberedUuid(200); + const created = objectOf( + folders.answer("post_folders", { + body: { uid, title: "New Folder Name" }, + }).body, + ); + expect(created).toMatchObject({ + uid, + url: `/dashboards/f/${uid}/new-folder-name`, + }); + expect(typeof created["id"]).toBe("number"); + expect(String(created["id"])).toHaveLength(16); + expect([ + 1655476044521472, 1656166527623168, 1656166745726976, + ]).not.toContain(created["id"]); + const renamed = objectOf( + folders.answer("put_folders_by_uuid", { + uuid: uid, + body: { title: "Moved Folder" }, + }).body, + ); + expect(renamed).toMatchObject({ + id: created["id"], + url: `/dashboards/f/${uid}/moved-folder`, + }); + expect( + folders.answer("get_folders_by_uuid", { uuid: uid }).body, + ).toMatchObject({ title: "Moved Folder", url: renamed["url"] }); + }); +}); diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts index eebf4b35c0..7d803995c2 100644 --- a/src/application/environment/environmentStateWriteAnswer.ts +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -30,6 +30,8 @@ import { } from "../../domain/analysisView/environmentStateWriteRules.js"; import { formatClock, + generatedIntegerDigits, + seededInteger, seededLike, spelledLike, } from "../../domain/analysisView/environmentStateWriteValues.js"; @@ -212,14 +214,14 @@ export const generator = ( ); const unused = ( field: string, - make: (attempt: number) => string, - ): Result => { + make: (attempt: number) => JsonValue, + ): Result => { const held = new Set( ofType.map((entity) => scalarText(entity.fields[field])), ); for (let attempt = 0; attempt < GENERATION_ATTEMPTS; attempt += 1) { const value = make(attempt); - if (!held.has(value)) return ok(value); + if (!held.has(scalarText(value))) return ok(value); } return err( `Could not generate a unique value for ${field} after ${String(GENERATION_ATTEMPTS)} attempts without colliding with the held state.`, @@ -267,7 +269,9 @@ export const generator = ( ); case "unique": return unused(rule.field, (attempt) => - seededLike(seed(attempt), exemplar), + generatedIntegerDigits(exemplar) === undefined + ? seededLike(seed(attempt), exemplar) + : seededInteger(seed(attempt), exemplar), ); case "renewed": return ok( diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index c24249fb96..2a9f21b129 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -219,6 +219,7 @@ export class EnvironmentStateWriteSimulation { sequences: new Map(), steady: new Set(), unstamped: new Set(), + fixedWidth: new Map(), linked: link.collection, }); } diff --git a/src/domain/analysisView/environmentStateDerivedWrites.test.ts b/src/domain/analysisView/environmentStateDerivedWrites.test.ts new file mode 100644 index 0000000000..8fec6c4f91 --- /dev/null +++ b/src/domain/analysisView/environmentStateDerivedWrites.test.ts @@ -0,0 +1,517 @@ +import { describe, expect, it } from "vitest"; + +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + BOOKMARK_ROWS, + FIXED_HELD_TAG_ROWS, + FIXED_SIBLING_TAG_ROWS, + FOLDER_ROWS, + goClock, + LATER_FIXED_SIBLING_TAG_ROWS, + MIDDLE_FIXED_TAG_UPDATE_ROWS, + MIXED_WIDTH_TAG_UPDATE_ROWS, + restoredAddressRows, + TAG_ROWS, + TAG_UPDATE_ROWS, + TWO_CLOCK_TAG_UPDATE_ROWS, + UNKNOWN_TAG_UPDATE_ROWS, +} from "../../../tests/fixtures/environmentStateDerivedValues.js"; +import { + environmentStateViewOf, + numberedUuid, +} from "../../../tests/fixtures/environmentStateWrites.js"; +import type { JsonValue } from "../jsonValue.js"; +import { isObject, type JsonObject } from "./environmentStateRules.js"; + +const planOf = (rows: readonly TimedCaptureRow[], tool: string) => { + const plan = environmentStateViewOf(timedCaptureEvidence(rows)).writes.find( + (candidate) => candidate.tool === tool, + ); + if (plan === undefined) throw new Error(`No plan for ${tool}`); + return plan; +}; + +const TRIMMED = { + kind: "iso", + separator: "T", + fraction: 9, + zone: "Z", + trimmed: true, +} as const; + +describe("later creates must retain the learned value rules", () => { + it("withholds a later create whose captured composed address is null", () => { + const snapshot = null; + const rows = BOOKMARK_ROWS.slice(0, 6).map( + (row, index): TimedCaptureRow => { + if (index !== 5) return row; + const [method, path, body, ...rest] = row; + if (!isObject(body)) throw new Error("Expected a bookmark response"); + return [ + method, + path, + { ...body, web_archive_snapshot_url: snapshot }, + ...rest, + ]; + }, + ); + expect(planOf(rows, "post_bookmarks")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it.each([ + { + name: "a string integer becomes a UUID", + exemplar: String, + later: numberedUuid(888), + }, + { + name: "a string integer becomes an equal-length token", + exemplar: String, + later: "a656166745726976", + }, + { + name: "a UUID becomes an equal-length opaque token", + exemplar: (id: number) => numberedUuid(400 + (id % 1000)), + later: "a".repeat(35) + "1", + }, + ])("withholds a create when $name", ({ exemplar, later }) => { + const changed = (value: JsonValue): JsonValue => + Array.isArray(value) + ? value.map(changed) + : isObject(value) + ? Object.fromEntries( + Object.entries(value).map(([field, child]) => [ + field, + field === "id" && typeof child === "number" + ? child === 1656166745726976 + ? later + : exemplar(child) + : changed(child), + ]), + ) + : value; + const rows = FOLDER_ROWS.map((row): TimedCaptureRow => { + const [method, path, body, ...rest] = row; + return body === undefined ? row : [method, path, changed(body), ...rest]; + }); + expect(planOf(rows, "post_folders")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); +}); + +describe("clock spellings an application derives", () => { + it("serves creates whose clocks drop trailing fraction zeros, as Go spells nanoseconds", () => { + expect(planOf(TAG_ROWS, "post_tags")).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "updatedAt", rule: "clock", format: TRIMMED }, + ]), + }); + }); + + it.each([ + ["2026-10-01T09:00:00.12345678Z", "2026-10-01T10:00:00.120000000Z"], + ["2026-10-01T09:00:00.120000000Z", "2026-10-01T10:00:00.12345678Z"], + ["2026-10-01T09:00:00.12345678Z", "2026-10-01T10:00:00.123456780Z"], + ["2026-10-01T09:00:00.123456780Z", "2026-10-01T10:00:00.12345678Z"], + ])( + "keeps a create's clock at a fixed width when a held value ends its fraction in a zero (%s, %s)", + (first, second) => { + const held = [first, second].map((updatedAt, index) => ({ + id: numberedUuid(index + 1), + name: `Held ${index + 1}`, + updatedAt, + })); + const created = { + id: numberedUuid(3), + name: "Heavy", + updatedAt: goClock(3, "374763655"), + }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/tags", held], + ...held.map((tag): TimedCaptureRow => [ + "GET", + `/api/tags/${tag.id}`, + tag, + ]), + ["POST", "/api/tags", created, 201, { name: "Heavy" }], + ["GET", `/api/tags/${created.id}`, created], + ]; + expect(planOf(rows, "post_tags").rules).toContainEqual({ + field: "updatedAt", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 9, zone: "Z" }, + }); + }, + ); + + it("keeps a create's sibling clock at a fixed width when a later call trims one clock and shows the sibling's trailing zero", () => { + const rules = planOf(FIXED_SIBLING_TAG_ROWS, "post_tags").rules; + expect(rules).toContainEqual({ + field: "updatedAt", + rule: "clock", + format: TRIMMED, + }); + expect(rules).toContainEqual({ + field: "createdAt", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 9, zone: "Z" }, + }); + }); + + it("serves updates whose clock a later call shows trimmed", () => { + expect(planOf(TAG_UPDATE_ROWS, "patch_tags_by_uuid")).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "updatedAt", rule: "clock", format: TRIMMED }, + ]), + }); + }); + + it("learns no clock spelling from an update to an entity an unmodelled write left unknown", () => { + expect(planOf(UNKNOWN_TAG_UPDATE_ROWS, "patch_tags_by_uuid")).toMatchObject( + { + served: true, + rules: expect.arrayContaining([ + { + field: "updatedAt", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 9, zone: "Z" }, + }, + ]), + }, + ); + }); + + it("serves updates whose two clocks a later call shows trimmed for one only", () => { + expect( + planOf(TWO_CLOCK_TAG_UPDATE_ROWS, "patch_tags_by_uuid"), + ).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "updatedAt", rule: "clock", format: TRIMMED }, + { field: "syncedAt", rule: "clock", format: TRIMMED }, + ]), + }); + }); +}); + +describe("fixed-width clock evidence", () => { + const FIXED = { + kind: "iso", + separator: "T", + fraction: 9, + zone: "Z", + } as const; + + it.each([1, 2])( + "withholds a shorter update clock when held entity %s proves a fixed width", + (heldId) => { + const fixedHeld = (value: JsonValue): JsonValue => + Array.isArray(value) + ? value.map(fixedHeld) + : isObject(value) && value.id === numberedUuid(heldId) + ? { ...value, updatedAt: "2026-10-01T09:00:00.120000000Z" } + : value; + const rows = TAG_UPDATE_ROWS.map((row, index): TimedCaptureRow => { + const [method, path, body, ...rest] = row; + return index < 3 && body !== undefined + ? [method, path, fixedHeld(body), ...rest] + : row; + }); + expect(planOf(rows, "patch_tags_by_uuid")).toMatchObject({ + served: false, + reason: "unexplained-change", + rules: expect.arrayContaining([ + { field: "updatedAt", rule: "clock", format: FIXED }, + ]), + }); + }, + ); + + it("serves updates whose later call trims one clock and shows the other's trailing zero", () => { + expect( + planOf(MIXED_WIDTH_TAG_UPDATE_ROWS, "patch_tags_by_uuid"), + ).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "updatedAt", rule: "clock", format: TRIMMED }, + { field: "syncedAt", rule: "clock", format: FIXED }, + ]), + }); + }); + + it("keeps a create's clock at a fixed width an earlier later call showed, whatever a third shows", () => { + expect( + planOf(LATER_FIXED_SIBLING_TAG_ROWS, "post_tags").rules, + ).toContainEqual({ field: "createdAt", rule: "clock", format: FIXED }); + }); + + it("keeps a create's clock at a fixed width a held entity showed while its sibling is learned trimmed", () => { + const rules = planOf(FIXED_HELD_TAG_ROWS, "post_tags").rules; + expect(rules).toContainEqual({ + field: "updatedAt", + rule: "clock", + format: TRIMMED, + }); + expect(rules).toContainEqual({ + field: "createdAt", + rule: "clock", + format: FIXED, + }); + }); + + it("withholds updates whose clock a later call trims after an earlier one showed a trailing zero", () => { + expect( + planOf(MIDDLE_FIXED_TAG_UPDATE_ROWS, "patch_tags_by_uuid"), + ).toMatchObject({ + served: false, + reason: "unexplained-change", + rules: expect.arrayContaining([ + { field: "updatedAt", rule: "clock", format: FIXED }, + ]), + }); + }); +}); + +describe("values an application composes or generates", () => { + it("withholds bookmark creates a later call answers with a snapshot address the composition does not spell", () => { + const special = "https://web.archive.org/web/special"; + const rows = BOOKMARK_ROWS.flatMap((row): TimedCaptureRow[] => { + const [method, path, body, ...rest] = row; + if (method === "GET" && path === "/api/bookmarks/4") return []; + return method === "POST" && isObject(body) && body.id === 4 + ? [ + [ + method, + path, + { ...body, web_archive_snapshot_url: special }, + ...rest, + ], + ] + : [row]; + }).slice(0, -1); + expect(planOf(rows, "post_bookmarks")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("serves creates whose snapshot address joins the creation time and the sent URL, accepting any URL", () => { + expect(planOf(BOOKMARK_ROWS, "post_bookmarks")).toMatchObject({ + served: true, + body: { accepted: ["url", "title"], fixed: [] }, + rules: expect.arrayContaining([ + { + field: "web_archive_snapshot_url", + rule: "composed", + parts: [ + { text: "https://web.archive.org/web/" }, + { from: "date_added", as: "clock", pattern: "YYYYMMDDhhmmss" }, + { text: "/" }, + { from: "url", as: "value" }, + ], + }, + ]), + }); + }); + + it("serves folder creates and renames whose address follows the uid and the title's slug, and whose numeric id is generated", () => { + const url = { + field: "url", + rule: "composed", + parts: [ + { text: "/dashboards/f/" }, + { from: "uid", as: "value" }, + { text: "/" }, + { from: "title", as: "slug" }, + ], + }; + expect(planOf(FOLDER_ROWS, "post_folders")).toMatchObject({ + served: true, + body: { accepted: ["uid", "title"], fixed: [] }, + rules: expect.arrayContaining([{ field: "id", rule: "unique" }, url]), + }); + expect(planOf(FOLDER_ROWS, "put_folders_by_uuid")).toMatchObject({ + served: true, + rules: expect.arrayContaining([url]), + }); + }); + + it("withholds renames whose address a later call composes when an earlier call left it otherwise", () => { + expect( + planOf(restoredAddressRows(true), "put_folders_by_uuid"), + ).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + expect.objectContaining({ field: "url", rule: "composed" }), + ]), + }); + // The first rename left its address as it was, which the composition would rebuild. + expect( + planOf(restoredAddressRows(false), "put_folders_by_uuid"), + ).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it.each([null, { path: "/legacy" }])( + "withholds a later composition whose parts an earlier rename could not spell (%j)", + (base) => { + const first = { id: 1, title: "First Folder", base, url: "/legacy" }; + const renamedFirst = { ...first, title: "Renamed First Folder" }; + const second = { + id: 2, + title: "Second Folder", + base: "https://example.test/folders", + url: "https://example.test/folders/second-folder", + }; + const renamedSecond = { + ...second, + title: "Renamed Second Folder", + url: "https://example.test/folders/renamed-second-folder", + }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/folders", [first, second]], + ["GET", "/api/folders/1", first], + ["GET", "/api/folders/2", second], + [ + "PUT", + "/api/folders/1", + renamedFirst, + 200, + { title: renamedFirst.title }, + ], + ["GET", "/api/folders/1", renamedFirst], + [ + "PUT", + "/api/folders/2", + renamedSecond, + 200, + { title: renamedSecond.title }, + ], + ["GET", "/api/folders/2", renamedSecond], + ]; + expect(planOf(rows, "put_folders_by_id")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }, + ); +}); + +/** The folder rows with each folder's numeric id repeated where `mirror` puts it, and in each create's body when `sent`. */ +const mirrored = ( + mirror: (id: number) => JsonObject, + sent = false, +): TimedCaptureRow[] => { + const repeated = (value: JsonValue): JsonValue => + Array.isArray(value) + ? value.map(repeated) + : isObject(value) && typeof value.id === "number" + ? { ...value, ...mirror(value.id) } + : value; + return FOLDER_ROWS.map((row): TimedCaptureRow => { + const [method, path, body, ...rest] = row; + if (body === undefined) return row; + const answered = repeated(body); + const [status, request] = rest; + return sent && + method === "POST" && + isObject(answered) && + status !== undefined && + isObject(request) + ? [ + method, + path, + answered, + status, + { ...request, ...mirror(Number(answered.id)) }, + ] + : [method, path, answered, ...rest]; + }); +}; + +describe("generated integers another field repeats", () => { + it("withholds folder creates a later call answers with a UUID where the exemplar generated an integer", () => { + const later = 1656166745726976; + const answered = numberedUuid(203); + const swap = (value: JsonValue): JsonValue => + Array.isArray(value) + ? value.map(swap) + : isObject(value) && value.id === later + ? { ...value, id: answered } + : value; + const rows = FOLDER_ROWS.map((row): TimedCaptureRow => { + const [method, path, body, ...rest] = row; + return body === undefined ? row : [method, path, swap(body), ...rest]; + }); + expect(planOf(rows, "post_folders")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("withholds folder creates whose generated numeric id another field repeats, which seeding cannot keep", () => { + expect( + planOf( + mirrored((id) => ({ folderId: id })), + "post_folders", + ), + ).toMatchObject({ served: false, reason: "unexplained-change" }); + }); + + it("withholds folder creates whose generated numeric id a number nested in another field repeats", () => { + expect( + planOf( + mirrored((id) => ({ metadata: { serial: id } })), + "post_folders", + ), + ).toMatchObject({ served: false, reason: "unexplained-change" }); + }); + + it.each([ + [ + "an echoed string of its digits", + (id: number) => ({ serialText: String(id) }), + true, + ], + [ + "a string another rule generates around its digits", + (id: number) => ({ ref: `folder-${id}` }), + false, + ], + ])( + "withholds folder creates whose generated numeric id %s repeats", + (_, mirror, sent) => { + expect(planOf(mirrored(mirror, sent), "post_folders")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }, + ); + + it("serves folder creates whose string of the id's digits a template rebuilds from it", () => { + expect( + planOf( + mirrored((id) => ({ idText: String(id) })), + "post_folders", + ), + ).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "id", rule: "unique" }, + { field: "idText", rule: "template", from: "id" }, + ]), + }); + }); +}); diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index 87e92de8a7..c01c556b06 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -9,6 +9,7 @@ import type { } from "./analysisView.js"; import { EnvironmentStateAccumulator } from "./EnvironmentStateAccumulator.js"; import { isClockZone } from "./environmentStateWriteValues.js"; +import { CLOCK_PATTERNS } from "./environmentStateWriteClockFormats.js"; import { groupCaptureRecords, templateRulesSchema, @@ -97,6 +98,7 @@ const clockFormatSchema = z.discriminatedUnion("kind", [ separator: z.enum(["T", " "]), fraction: z.number().int().nonnegative().max(9), zone: z.string().refine(isClockZone), + trimmed: z.literal(true).exactOptional(), }), z.strictObject({ kind: z.literal("epoch"), unit: z.enum(["s", "ms"]) }), ]); @@ -256,6 +258,26 @@ const fieldRuleSchema = z.discriminatedUnion("rule", [ z.strictObject({ ...ruled, rule: z.literal("path"), from: z.string() }), z.strictObject({ ...ruled, rule: z.literal("slug"), from: z.string() }), z.strictObject({ ...ruled, rule: z.literal("template"), from: z.string() }), + z.strictObject({ + ...ruled, + rule: z.literal("composed"), + parts: z + .array( + z.union([ + z.strictObject({ text: z.string().min(1) }), + z.strictObject({ + from: z.string(), + as: z.enum(["value", "slug"]), + }), + z.strictObject({ + from: z.string(), + as: z.literal("clock"), + pattern: z.enum(CLOCK_PATTERNS), + }), + ]), + ) + .min(1), + }), z.strictObject({ ...ruled, rule: z.literal("increment") }), z.strictObject({ ...ruled, diff --git a/src/domain/analysisView/environmentStateWriteApply.test.ts b/src/domain/analysisView/environmentStateWriteApply.test.ts index 46ed37f429..bce091a9c2 100644 --- a/src/domain/analysisView/environmentStateWriteApply.test.ts +++ b/src/domain/analysisView/environmentStateWriteApply.test.ts @@ -676,3 +676,81 @@ it("checks a member stored only under another name against the type of the field expect(created({ not: "a string" }).ok).toBe(false); expect(created(null).ok).toBe(false); }); + +const ADDRESS: FieldRule = { + field: "url", + rule: "composed", + parts: [ + { text: "/dashboards/f/" }, + { from: "uid", as: "value" }, + { text: "/" }, + { from: "title", as: "slug" }, + ], +}; +const UNSPELLED = + 'The write composes "url" from "uid", "title", and the entity holds no value one of them can spell.'; + +it("rebuilds an updated composed address, and refuses one whose entity holds no field a part reads", () => { + const plan: Plan = { + rules: [{ field: "title", rule: "echo" }, ADDRESS], + body: { + accepted: ["title"], + fixed: [], + required: [], + identity: "uid", + kinds: [], + }, + }; + const rename = (before: JsonObject) => + updatedFields( + plan, + { before, observed: before }, + { ...callWith({ title: "Moved Folder" }, { item: "7" }), id: "7" }, + generator().generate, + ); + expect( + rename({ id: 7, uid: "f-7", title: "Old", url: "/dashboards/f/f-7/old" }), + ).toEqual( + ok({ + id: 7, + uid: "f-7", + title: "Moved Folder", + url: "/dashboards/f/f-7/moved-folder", + }), + ); + expect(rename({ id: 7, title: "Old", url: "/dashboards/f/f-7/old" })).toEqual( + err(UNSPELLED), + ); +}); + +it("refuses a create whose composed address reads a member the call does not send", () => { + const plan: Plan = { + rules: [ + { field: "uid", rule: "echo" }, + { field: "title", rule: "echo" }, + ADDRESS, + ], + body: { + accepted: ["uid", "title"], + fixed: [], + required: ["title"], + identity: null, + kinds: [], + }, + }; + const observed: JsonObject = { + uid: "f-1", + title: "Recorded", + url: "/dashboards/f/f-1/recorded", + }; + const create = (body: JsonObject) => + createdFields(plan, observed, callWith(body), generator().generate); + expect(create({ uid: "f-2", title: "New Folder" })).toEqual( + ok({ + uid: "f-2", + title: "New Folder", + url: "/dashboards/f/f-2/new-folder", + }), + ); + expect(create({ title: "New Folder" })).toEqual(err(UNSPELLED)); +}); diff --git a/src/domain/analysisView/environmentStateWriteApply.ts b/src/domain/analysisView/environmentStateWriteApply.ts index 7a57b41d30..f980e1138b 100644 --- a/src/domain/analysisView/environmentStateWriteApply.ts +++ b/src/domain/analysisView/environmentStateWriteApply.ts @@ -24,6 +24,7 @@ import { stampProblem, nestedClockValue, } from "./environmentStateWriteStamps.js"; +import { composedValue } from "./environmentStateWriteComposed.js"; import { integerOf, readClock, @@ -260,13 +261,13 @@ const templated = ( : undefined; }; -/** Apply the rules that derive from fields already written or the call: slugs, templates and stamps. */ +/** Apply the rules that derive from fields already written or the call: slugs, templates, stamps and composed strings; why a composed string cannot be rebuilt, if one cannot. */ const derive = ( rules: readonly FieldRule[], observed: JsonObject, written: JsonObject, sent: JsonObject, -): void => { +): string | undefined => { for (const rule of rules) { if (rule.rule === "slug") { const from = sent[rule.from]; @@ -278,6 +279,19 @@ const derive = ( else if (rule.rule === "derived" && Object.hasOwn(sent, rule.member)) written[rule.field] = rule.value; } + // Composed strings read the fields the rules above wrote. + for (const rule of rules) + if (rule.rule === "composed") { + const value = composedValue(rule.parts, written); + if (value === undefined) { + const sources = new Set( + rule.parts.flatMap((part) => ("from" in part ? [part.from] : [])), + ); + return `The write composes ${JSON.stringify(rule.field)} from ${listed([...sources])}, and the entity holds no value one of them can spell.`; + } + written[rule.field] = value; + } + return undefined; }; /** Members stored only under another name, with the value of the field each is copied whole into, for type checks. */ @@ -313,8 +327,8 @@ export const createdFields = ( if (value.value !== undefined) written[rule.field] = value.value; else if (rule.rule === "echo") delete written[rule.field]; } - derive(plan.rules, observed, written, call.body ?? {}); - return ok(written); + const unspelled = derive(plan.rules, observed, written, call.body ?? {}); + return unspelled === undefined ? ok(written) : err(unspelled); }; /** Rules an update gives a new value on every call. */ @@ -384,7 +398,7 @@ export const updatedFields = ( if (value.value !== undefined) written[rule.field] = value.value; } } - derive( + const unspelled = derive( plan.rules.filter( (rule) => rule.rule !== "slug" || Object.hasOwn(sent, rule.from), ), @@ -392,7 +406,7 @@ export const updatedFields = ( written, sent, ); - return ok(written); + return unspelled === undefined ? ok(written) : err(unspelled); }; /** Why a document write's body does not fit what the capture showed: members its plan accepts, as the JSON types the document holds, a member a replacing write dropped keeping the type the initial document showed. */ diff --git a/src/domain/analysisView/environmentStateWriteCalls.ts b/src/domain/analysisView/environmentStateWriteCalls.ts index f8ad8ad29a..6b0402f909 100644 --- a/src/domain/analysisView/environmentStateWriteCalls.ts +++ b/src/domain/analysisView/environmentStateWriteCalls.ts @@ -3,6 +3,7 @@ import { digestCanonicalValue } from "../canonicalDigest.js"; import type { JsonValue } from "../jsonValue.js"; import { same, type JsonObject } from "./environmentStateRules.js"; import { storedKinds } from "./environmentStateWriteApply.js"; +import { composedValue } from "./environmentStateWriteComposed.js"; import type { explainUpdate } from "./environmentStateWriteExplain.js"; import type { FieldRule, @@ -15,6 +16,7 @@ import { integerOf, slugOf } from "./environmentStateWriteValues.js"; export const unsentChange = (rule: FieldRule): boolean => rule.rule === "increment" || rule.rule === "slug" || + rule.rule === "composed" || rule.rule === "stamp" || rule.rule === "derived" || rule.rule === "set"; @@ -81,6 +83,14 @@ const observedValue = (value: JsonValue): ObservedValue => ? { kind: "structured", digest: digestCanonicalValue(value) } : { kind: "scalar", value }; +/** The scalar fields a call showed, as a composed rule reads them. */ +const shownScalars = (shown: ReadonlyMap): JsonObject => + Object.fromEntries( + [...shown].flatMap(([field, observed]) => + observed.kind === "scalar" ? [[field, observed.value]] : [], + ), + ); + const sameObserved = (left: ObservedValue, right: ObservedValue): boolean => left.kind === "structured" ? right.kind === "structured" && left.digest === right.digest @@ -129,6 +139,13 @@ export const contradictsEarlierCalls = ( if (rule.rule === "stamp") return !sameObserved(observation, expected!); if (rule.rule === "derived") return sent.has(rule.member) && !sameObserved(observation, expected!); + if (rule.rule === "composed") { + const value = composedValue(rule.parts, shownScalars(shown)); + return ( + value === undefined || + !sameObserved(observation, observedValue(value)) + ); + } const from = rule.rule === "slug" ? sent.get(rule.from) : undefined; return ( typeof from === "string" && diff --git a/src/domain/analysisView/environmentStateWriteChanges.ts b/src/domain/analysisView/environmentStateWriteChanges.ts index a13b2f9f52..1786655e6f 100644 --- a/src/domain/analysisView/environmentStateWriteChanges.ts +++ b/src/domain/analysisView/environmentStateWriteChanges.ts @@ -3,6 +3,7 @@ import { isObject, same, type JsonObject } from "./environmentStateRules.js"; import type { FieldRule, WriteCall } from "./environmentStateWriteRules.js"; import { integerOf, readClock } from "./environmentStateWriteValues.js"; import { freshUuid, renewedRule } from "./environmentStateWriteGenerated.js"; +import { composedChange } from "./environmentStateWriteComposed.js"; import { derivedRule, freshToken, @@ -59,6 +60,10 @@ export const changeRule = ( return ( stampRule(evidence, field) ?? derivedRule(evidence, field) ?? + composedChange(field, { + ...evidence, + sent: evidence.call.body, + }) ?? actionRule(evidence, field) ); }; diff --git a/src/domain/analysisView/environmentStateWriteClockFormats.test.ts b/src/domain/analysisView/environmentStateWriteClockFormats.test.ts new file mode 100644 index 0000000000..deff640d20 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteClockFormats.test.ts @@ -0,0 +1,111 @@ +import { expect, it } from "vitest"; +import type { JsonValue } from "../jsonValue.js"; +import { + clockDigits, + fittedClockFormat, + trimmedLike, +} from "./environmentStateWriteClockFormats.js"; +import type { ClockFormat } from "./environmentStateWriteValues.js"; + +const iso = ( + separator: "T" | " ", + fraction: number, + zone: string, +): ClockFormat => ({ kind: "iso", separator, fraction, zone }); + +const WITH_MS = Date.UTC(2026, 2, 4, 5, 6, 7) + 123; + +const TRIMMED_9: ClockFormat = { + kind: "iso", + separator: "T", + fraction: 9, + zone: "Z", + trimmed: true, +}; + +it.each< + [string, ClockFormat, string, string | undefined, ClockFormat | undefined] +>([ + [ + "the format's own spelling", + iso("T", 9, "Z"), + "2026-03-04T05:06:07.123456789Z", + "2026-03-04T05:06:07.123456781Z", + iso("T", 9, "Z"), + ], + [ + "a shorter trimmed fraction", + iso("T", 9, "Z"), + "2026-03-04T05:06:07.12345678Z", + "2026-03-04T05:06:07.123456781Z", + TRIMMED_9, + ], + [ + "a longer one than a trimmed exemplar", + iso("T", 8, "Z"), + "2026-03-04T05:06:07.123456781Z", + "2026-03-04T05:06:07.12345678Z", + TRIMMED_9, + ], + [ + "no fraction beside a trimmed one", + iso("T", 9, "Z"), + "2026-03-04T05:06:07Z", + "2026-03-04T05:06:07.123456781Z", + TRIMMED_9, + ], + [ + "a fraction ending in a zero", + iso("T", 8, "Z"), + "2026-03-04T05:06:07.123456780Z", + "2026-03-04T05:06:07.12345678Z", + undefined, + ], + [ + "an exemplar ending in a zero", + iso("T", 3, "Z"), + "2026-03-04T05:06:07.123456Z", + "2026-03-04T05:06:07.120Z", + undefined, + ], + [ + "another zone", + iso("T", 9, "Z"), + "2026-03-04T05:06:07.1234567+00:00", + "2026-03-04T05:06:07.123456781Z", + undefined, + ], + [ + "a shorter value under a trimmed format", + TRIMMED_9, + "2026-03-04T05:06:07.1Z", + undefined, + TRIMMED_9, + ], +])("fits %s to a clock format", (_name, format, value, exemplar, expected) => { + expect(fittedClockFormat(format, value, exemplar)).toEqual(expected); +}); + +it("trims a sibling clock spelled alike unless its own value shows a fixed width", () => { + expect( + trimmedLike(iso("T", 9, "Z"), TRIMMED_9, "2026-03-04T05:06:07.123456789Z"), + ).toEqual(TRIMMED_9); + expect( + trimmedLike(iso("T", 9, "Z"), TRIMMED_9, "2026-03-04T05:06:07.123456780Z"), + ).toEqual(iso("T", 9, "Z")); + expect( + trimmedLike(iso(" ", 9, "Z"), TRIMMED_9, "2026-03-04 05:06:07.123456789Z"), + ).toEqual(iso(" ", 9, "Z")); +}); + +it.each<[JsonValue, string, string | undefined]>([ + ["2026-03-04T05:06:07.123Z", "YYYYMMDDhhmmss", "20260304050607"], + ["2026-03-04T05:06:07Z", "YYYYMMDD", "20260304"], + ["2026-03-04T23:30:00-05:00", "YYYY-MM-DD", "2026-03-04"], + [WITH_MS, "YYYY/MM/DD", "2026/03/04"], + ["not a clock", "YYYYMMDD", undefined], +])("spells the clock %j's wall clock in %s", (value, pattern, expected) => { + expect(clockDigits(value, pattern as Parameters[1])).toBe( + expected, + ); +}); diff --git a/src/domain/analysisView/environmentStateWriteClockFormats.ts b/src/domain/analysisView/environmentStateWriteClockFormats.ts new file mode 100644 index 0000000000..883b752df5 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteClockFormats.ts @@ -0,0 +1,164 @@ +import type { JsonValue } from "../jsonValue.js"; +import { + readClock, + zoneMinutesForFormat, + type ClockFormat, + type ClockReading, +} from "./environmentStateWriteValues.js"; + +const MINUTE_MS = 60_000; + +const pad = (value: number, width: number): string => + String(value).padStart(width, "0"); + +/** Whether a parsed clock fits a learned format without widening it or learning a new spelling. */ +export const matchesClockFormat = ( + format: ClockFormat, + reading: ClockReading | undefined, +): boolean => { + if (reading === undefined) return false; + const spelled = reading.format; + if (format.kind === "epoch") + return spelled.kind === "epoch" && spelled.unit === format.unit; + return ( + spelled.kind === "iso" && + spelled.separator === format.separator && + spelled.zone === format.zone && + (format.trimmed === true + ? !reading.fixedWidth && spelled.fraction <= format.fraction + : spelled.fraction === format.fraction) + ); +}; + +/** + * The format that spells both a format's values and another clock value: the format itself, or a trimmed one when the + * two differ only in how many fraction digits they show and neither ends a fraction in a zero. `exemplar` is the value + * the format was learned from, which shows whether its own width is fixed. + */ +export const fittedClockFormat = ( + format: ClockFormat, + value: JsonValue | undefined, + exemplar: JsonValue | undefined, +): ClockFormat | undefined => { + const reading = readClock(value); + if (reading === undefined) return undefined; + if (matchesClockFormat(format, reading)) return format; + const spelled = reading.format; + if ( + format.kind !== "iso" || + spelled.kind !== "iso" || + spelled.separator !== format.separator || + spelled.zone !== format.zone || + reading.fixedWidth + ) + return undefined; + if (format.trimmed !== true) { + const learned = readClock(exemplar); + return learned === undefined || learned.fixedWidth + ? undefined + : { + ...format, + fraction: Math.max(format.fraction, spelled.fraction), + trimmed: true, + }; + } + return { ...format, fraction: spelled.fraction }; +}; + +/** Whether a value spells a clock as an ISO format does with a fraction ending in a zero, which no trimmed spelling shows. */ +export const showsFixedWidth = ( + format: ClockFormat, + value: JsonValue | undefined, +): boolean => { + const reading = readClock(value); + return ( + reading !== undefined && + reading.fixedWidth && + format.kind === "iso" && + reading.format.kind === "iso" && + reading.format.separator === format.separator && + reading.format.zone === format.zone + ); +}; + +/** Keep the first value showing each field as an ISO clock at a fixed width, which no trimmed spelling of the field may contradict. */ +export const noteFixedWidths = ( + seen: Map, + fields: Readonly>, +): void => { + for (const [field, value] of Object.entries(fields)) { + const reading = readClock(value); + if ( + reading?.fixedWidth === true && + reading.format.kind === "iso" && + !seen.has(field) + ) + seen.set(field, value); + } +}; + +/** A clock format trimmed as a sibling's is, when it is spelled alike and its own value does not show a fixed width. */ +export const trimmedLike = ( + format: ClockFormat, + trimmed: ClockFormat, + value: JsonValue | undefined, +): ClockFormat => { + const reading = readClock(value); + return format.kind === "iso" && + trimmed.kind === "iso" && + trimmed.trimmed === true && + format.trimmed !== true && + format.separator === trimmed.separator && + format.zone === trimmed.zone && + reading !== undefined && + !reading.fixedWidth + ? { + ...format, + fraction: Math.max(format.fraction, trimmed.fraction), + trimmed: true, + } + : format; +}; + +/** Calendar digits a composed string spells a clock in: date and time run together, or a date. */ +export type ClockPattern = + | "YYYYMMDDhhmmss" + | "YYYYMMDD" + | "YYYY-MM-DD" + | "YYYY/MM/DD"; + +export const CLOCK_PATTERNS: readonly ClockPattern[] = [ + "YYYYMMDDhhmmss", + "YYYYMMDD", + "YYYY-MM-DD", + "YYYY/MM/DD", +]; + +/** A clock value's wall clock in its own zone, in a calendar pattern; undefined when it is not a clock. */ +export const clockDigits = ( + value: JsonValue | undefined, + pattern: ClockPattern, +): string | undefined => { + const clock = readClock(value); + if (clock === undefined) return undefined; + const zone = + clock.format.kind === "iso" ? zoneMinutesForFormat(clock.format.zone) : 0; + const at = new Date(clock.ms + zone * MINUTE_MS); + const year = at.getUTCFullYear(); + if (!Number.isInteger(year) || year < 0 || year > 9999) return undefined; + const [y, m, d] = [ + pad(year, 4), + pad(at.getUTCMonth() + 1, 2), + pad(at.getUTCDate(), 2), + ]; + switch (pattern) { + case "YYYYMMDDhhmmss": + return `${y}${m}${d}${pad(at.getUTCHours(), 2)}${pad(at.getUTCMinutes(), 2)}${pad(at.getUTCSeconds(), 2)}`; + case "YYYYMMDD": + return `${y}${m}${d}`; + case "YYYY-MM-DD": + return `${y}-${m}-${d}`; + case "YYYY/MM/DD": + return `${y}/${m}/${d}`; + } +}; diff --git a/src/domain/analysisView/environmentStateWriteComposed.test.ts b/src/domain/analysisView/environmentStateWriteComposed.test.ts new file mode 100644 index 0000000000..c350b4f62a --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteComposed.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, it } from "vitest"; + +import type { JsonValue } from "../jsonValue.js"; +import type { JsonObject } from "./environmentStateRules.js"; +import { + composedChange, + composedValue, + createdComposer, +} from "./environmentStateWriteComposed.js"; +import type { FieldRule } from "./environmentStateWriteRules.js"; + +const CLOCK: FieldRule = { + field: "added", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 0, zone: "Z" }, +}; + +const bookmark = (url: string, added: string): JsonObject => ({ + url, + added, + modified: added, + snapshot: `https://web.archive.org/web/${added.replace(/\D/gu, "").slice(0, 14)}/${url}`, +}); + +const SEED = bookmark( + "https://ferries.example.test/timetable", + "2026-10-01T09:00:00Z", +); +const CREATED = bookmark( + "https://quarry.example.test/ledger", + "2026-10-17T18:00:03Z", +); +const ECHO_URL: FieldRule = { field: "url", rule: "echo" }; + +describe("strings a create composes from other fields", () => { + it("spells a snapshot address from the write clock's digits and the sent URL, as held entities do", () => { + const rule = createdComposer( + CREATED, + [ECHO_URL, CLOCK], + [SEED], + )("snapshot"); + expect(rule).toEqual({ + field: "snapshot", + rule: "composed", + parts: [ + { text: "https://web.archive.org/web/" }, + { from: "added", as: "clock", pattern: "YYYYMMDDhhmmss" }, + { text: "/" }, + { from: "url", as: "value" }, + ], + }); + expect( + composedValue(rule!.rule === "composed" ? rule!.parts : [], SEED), + ).toBe(SEED["snapshot"]); + }); + + it("composes nothing one call alone shows, nor what a held entity spells otherwise", () => { + expect( + createdComposer(CREATED, [ECHO_URL, CLOCK], [])("snapshot"), + ).toBeUndefined(); + const stale = { + ...SEED, + snapshot: "https://web.archive.org/web/20250101000000/x", + }; + expect( + createdComposer(CREATED, [ECHO_URL, CLOCK], [SEED, stale])("snapshot"), + ).toBeUndefined(); + }); + + it.each([ + ["before", (url: string) => `𐐀${url}`], + ["after", (url: string) => `${url}𐐀`], + ])( + "composes nothing from a value an astral letter joins %s it", + (_, label) => { + const labelled = (entity: JsonObject): JsonObject => ({ + ...entity, + label: label(String(entity["url"])), + }); + expect( + createdComposer( + labelled(CREATED), + [ECHO_URL, CLOCK], + [labelled(SEED)], + )("label"), + ).toBeUndefined(); + }, + ); + + it("takes two clocks a create stamps alike as one source, but not two members that send the same text", () => { + const modified: FieldRule = { ...CLOCK, field: "modified" }; + expect( + createdComposer(CREATED, [ECHO_URL, CLOCK, modified], [SEED])("snapshot"), + ).toMatchObject({ rule: "composed" }); + const folder = { + uid: "scratch-pier", + title: "Scratch Pier", + url: "/dashboards/f/scratch-pier/scratch-pier", + }; + const seed = { + uid: "lighthouse-rota", + title: "Lighthouse Rota", + url: "/dashboards/f/lighthouse-rota/lighthouse-rota", + }; + expect( + createdComposer( + folder, + [ + { field: "uid", rule: "echo" }, + { field: "title", rule: "echo" }, + ], + [seed], + )("url"), + ).toBeUndefined(); + }); +}); + +describe("strings an update composes from other fields", () => { + const before = { + uid: "scratch-alpha", + title: "Scratch Alpha Harbor", + url: "/dashboards/f/scratch-alpha/scratch-alpha-harbor", + }; + const after = { + ...before, + title: "Renamed Alpha Lantern", + url: "/dashboards/f/scratch-alpha/renamed-alpha-lantern", + }; + const change = (overrides: { + readonly after?: JsonObject; + readonly sent?: JsonObject; + readonly others?: readonly JsonObject[]; + }) => + composedChange("url", { + before, + after: overrides.after ?? after, + sent: overrides.sent ?? { title: "Renamed Alpha Lantern" }, + idField: "uid", + others: overrides.others ?? [], + }); + + it("spells an address from an unchanged uid and the sent title's slug, which spell it before the call too", () => { + expect(change({})).toEqual({ + field: "url", + rule: "composed", + parts: [ + { text: "/dashboards/f/" }, + { from: "uid", as: "value" }, + { text: "/" }, + { from: "title", as: "slug" }, + ], + }); + }); + + it("composes nothing from a member the application stored otherwise than sent, nor what another entity spells otherwise", () => { + expect( + change({ sent: { title: "RENAMED ALPHA LANTERN" } }), + ).toBeUndefined(); + expect( + change({ + others: [ + { uid: "other-folder", title: "Other Folder", url: "/f/other" }, + ], + }), + ).toBeUndefined(); + }); + + it("composes nothing the same parts do not spell before the call", () => { + expect( + change({ + after: { + ...after, + url: "/dashboards/f/scratch-alpha/renamed-alpha-lantern-1", + }, + }), + ).toBeUndefined(); + }); +}); + +describe("compositions serving could not keep", () => { + const stamped = (zone: string): FieldRule => ({ + field: "stamped", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 0, zone }, + }); + const dated = (url: string, added: string, at: string): JsonObject => ({ + url, + added, + stamped: at, + day: `https://archive.example.test/${added.slice(0, 10).replace(/-/gu, "")}/${url}`, + }); + const composeDay = (zone: string, created: string, seed: string) => + createdComposer( + dated( + "https://quarry.example.test/ledger", + "2026-10-17T12:00:03Z", + created, + ), + [ECHO_URL, CLOCK, stamped(zone)], + [ + dated( + "https://ferries.example.test/timetable", + "2026-10-01T09:00:00Z", + seed, + ), + ], + )("day"); + + it.each([ + ["+02:00", "2026-10-17T14:00:03+02:00", "2026-10-01T11:00:00+02:00"], + ["+05:00", "2026-10-17T17:00:03+05:00", "2026-10-01T14:00:00+05:00"], + ])( + "takes no date from two clocks in different zones, %s beside UTC, which a served instant near midnight tells apart", + (zone, created, seed) => { + expect(composeDay(zone, created, seed)).toBeUndefined(); + }, + ); + + it("still takes a date from two clocks the create stamps in one zone", () => { + expect( + composeDay("Z", "2026-10-17T12:00:03Z", "2026-10-01T09:00:00Z"), + ).toMatchObject({ rule: "composed" }); + }); + + it.each([null, 42])( + "composes nothing a held entity holds as %s while every part it reads is there", + (held) => { + const other = { + ...bookmark( + "https://ferries.example.test/rota", + "2026-10-02T09:00:00Z", + ), + snapshot: held, + }; + expect( + createdComposer(CREATED, [ECHO_URL, CLOCK], [SEED, other])("snapshot"), + ).toBeUndefined(); + const { snapshot: _, ...without } = other; + expect( + createdComposer( + CREATED, + [ECHO_URL, CLOCK], + [SEED, without], + )("snapshot"), + ).toMatchObject({ rule: "composed" }); + const folder = (url: JsonValue) => ({ + uid: "other-folder", + title: "Other Folder", + url, + }); + const before = { + uid: "scratch-alpha", + title: "Scratch Alpha Harbor", + url: "/dashboards/f/scratch-alpha/scratch-alpha-harbor", + }; + const rename = (others: readonly JsonObject[]) => + composedChange("url", { + before, + after: { + ...before, + title: "Renamed Alpha Lantern", + url: "/dashboards/f/scratch-alpha/renamed-alpha-lantern", + }, + sent: { title: "Renamed Alpha Lantern" }, + idField: "uid", + others, + }); + expect(rename([folder(held)])).toBeUndefined(); + const { url: __, ...unaddressed } = folder(held); + expect(rename([unaddressed])).toMatchObject({ rule: "composed" }); + }, + ); +}); + +it( + "indexes a wide body's repeated values once, and only when a string is left to compose", + { timeout: 5_000 }, + () => { + const n = 8_000; + const shared = `https://shared.example.test/${"x".repeat(64)}`; + const body: JsonObject = Object.fromEntries( + Array.from({ length: n }, (_, index) => [`e${String(index)}`, shared]), + ); + const rules: FieldRule[] = Array.from({ length: n }, (_, index) => ({ + field: `e${String(index)}`, + rule: "echo", + })); + let reads = 0; + const counted = new Proxy(body, { + get: (target, key) => { + reads += 1; + return Reflect.get(target, key); + }, + }); + const composer = createdComposer(counted, rules, []); + expect(reads).toBe(0); + expect(composer("e0")).toBeUndefined(); + expect(composer("e1")).toBeUndefined(); + // One read per source to index them, once, and one per composed field. + expect(reads).toBe(n + 2); + }, +); diff --git a/src/domain/analysisView/environmentStateWriteComposed.ts b/src/domain/analysisView/environmentStateWriteComposed.ts new file mode 100644 index 0000000000..0017aa905f --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteComposed.ts @@ -0,0 +1,274 @@ +import type { JsonValue } from "../jsonValue.js"; +import { same, scalarText, type JsonObject } from "./environmentStateRules.js"; +import { COPY_MIN_LENGTH } from "./environmentStateWriteCopies.js"; +import { GENERATED } from "./environmentStateWriteGenerated.js"; +import type { ComposedPart, FieldRule } from "./environmentStateWriteRules.js"; +import { + joinsAfter, + joinsBefore, + readClock, + slugOf, + zoneMinutesForFormat, +} from "./environmentStateWriteValues.js"; +import { + CLOCK_PATTERNS, + clockDigits, +} from "./environmentStateWriteClockFormats.js"; + +type SourcePart = Exclude; + +/** Text a field's value offers a composed string, and the part that spells it. */ +interface Candidate { + readonly token: string; + readonly part: SourcePart; +} + +/** Rules whose field holds a value the call or serving decides, which a created composed string may show. */ +const SOURCES: ReadonlySet = new Set([ + ...GENERATED, + "echo", + "copy", + "path", + "slug", + "clock", +]); + +/** The text a part spells from written fields; undefined when its field holds nothing it can spell. */ +export const composedPartText = ( + part: ComposedPart, + fields: JsonObject, +): string | undefined => { + if ("text" in part) return part.text; + const value = fields[part.from]; + if (part.as === "clock") return clockDigits(value, part.pattern); + if (part.as === "value") return scalarText(value); + return typeof value === "string" ? slugOf(value) : undefined; +}; + +/** A composed string spelled from written fields, or undefined when a part has nothing to spell. */ +export const composedValue = ( + parts: readonly ComposedPart[], + fields: JsonObject, +): string | undefined => { + let text = ""; + for (const part of parts) { + const spelled = composedPartText(part, fields); + if (spelled === undefined) return undefined; + text += spelled; + } + return text; +}; + +/** The texts a field's value offers: itself and its slug when long enough to show it, and a clock's calendar digits. */ +const candidatesOf = ( + field: string, + value: JsonValue | undefined, + shortest: number, +): Candidate[] => { + const candidates: Candidate[] = []; + const text = scalarText(value); + if (text !== undefined && text.length >= shortest) + candidates.push({ token: text, part: { from: field, as: "value" } }); + if (typeof value === "string") { + const slug = slugOf(value); + if (slug !== value && slug.length >= COPY_MIN_LENGTH) + candidates.push({ token: slug, part: { from: field, as: "slug" } }); + } + if (readClock(value) !== undefined) + for (const pattern of CLOCK_PATTERNS) { + const digits = clockDigits(value, pattern); + if (digits !== undefined) + candidates.push({ + token: digits, + part: { from: field, as: "clock", pattern }, + }); + } + return candidates; +}; + +/** Candidates by their text, and the lengths those texts take, longest first: looked up per position rather than searched. */ +interface CandidateIndex { + readonly byToken: ReadonlyMap; + readonly lengths: readonly number[]; +} + +const indexed = (candidates: readonly Candidate[]): CandidateIndex => { + const byToken = new Map(); + for (const candidate of candidates) { + const same = byToken.get(candidate.token); + if (same === undefined) byToken.set(candidate.token, [candidate]); + else same.push(candidate); + } + const lengths = [...new Set(candidates.map(({ token }) => token.length))]; + return { byToken, lengths: lengths.sort((left, right) => right - left) }; +}; + +/** The longest candidates whose text starts at a position and ends at a delimiter. */ +const matchesAt = ( + text: string, + at: number, + index: CandidateIndex, +): readonly Candidate[] => { + if (joinsBefore(text, at)) return []; + for (const length of index.lengths) { + if (at + length > text.length || joinsAfter(text, at + length)) continue; + const matches = index.byToken.get(text.slice(at, at + length)); + if (matches !== undefined) return matches; + } + return []; +}; + +/** + * Literal text around the longest delimited candidate at each position, left to right; undefined when none shows, or + * when one text comes from two sources serving could tell apart. `alike` names parts serving fills with one value. + */ +const segmented = ( + text: string, + index: CandidateIndex, + alike: (left: SourcePart, right: SourcePart) => boolean, +): ComposedPart[] | undefined => { + const parts: ComposedPart[] = []; + let literal = ""; + let shown = false; + for (let at = 0; at < text.length;) { + const matches = matchesAt(text, at, index); + const match = matches[0]; + if (match === undefined) { + literal += text[at]; + at += 1; + continue; + } + if (matches.some((other) => !alike(match.part, other.part))) + return undefined; + if (literal !== "") parts.push({ text: literal }); + literal = ""; + parts.push(match.part); + shown = true; + at += match.token.length; + } + if (literal !== "") parts.push({ text: literal }); + return shown ? parts : undefined; +}; + +const samePart = (left: SourcePart, right: SourcePart): boolean => + left.from === right.from && + left.as === right.as && + (left.as !== "clock" || + right.as !== "clock" || + left.pattern === right.pattern); + +/** How many entities holding the field and what its parts read spell it as the parts do, or undefined when one holds it otherwise, a non-string value included. */ +const heldSpellings = ( + parts: readonly ComposedPart[], + field: string, + held: readonly JsonObject[], +): number | undefined => { + let agreeing = 0; + for (const entity of held) { + if (!Object.hasOwn(entity, field)) continue; + const spelled = composedValue(parts, entity); + if (spelled === undefined) continue; + if (spelled !== entity[field]) return undefined; + agreeing += 1; + } + return agreeing; +}; + +/** + * For a created entity, each string field rebuilt from literal text and the fields other rules write, when it shows + * at least one and the type's other entities spell it the same way, at least one of them: one call alone cannot show + * which text is literal. + */ +export const createdComposer = ( + entity: JsonObject, + rules: readonly FieldRule[], + held: readonly JsonObject[], +): ((field: string) => FieldRule | undefined) => { + // Serving stamps every clock a create writes with the one write clock, so their digits in one zone are one source. + const zones = new Map( + rules.flatMap((rule) => + rule.rule === "clock" + ? [ + [ + rule.field, + rule.format.kind === "iso" + ? zoneMinutesForFormat(rule.format.zone) + : 0, + ] as const, + ] + : [], + ), + ); + const alike = (left: SourcePart, right: SourcePart): boolean => + samePart(left, right) || + (left.as === "clock" && + right.as === "clock" && + left.pattern === right.pattern && + zones.has(left.from) && + zones.get(left.from) === zones.get(right.from)); + let index: CandidateIndex | undefined; + return (field) => { + const text = entity[field]; + if (typeof text !== "string") return undefined; + index ??= indexed( + rules.flatMap((rule) => + SOURCES.has(rule.rule) + ? candidatesOf( + rule.field, + entity[rule.field], + // A generated value is short and still the call's own, as a template's is. + GENERATED.has(rule.rule) ? 1 : COPY_MIN_LENGTH, + ) + : [], + ), + ); + const parts = segmented(text, index, alike); + return parts === undefined || + parts.some((part) => "from" in part && part.from === field) || + !((heldSpellings(parts, field, held) ?? 0) > 0) + ? undefined + : { field, rule: "composed", parts }; + }; +}; + +/** + * An unsent string field an update rebuilt from literal text and fields serving knows, members stored as sent and those + * the call left as they were: the same parts spell it before the call and on the type's other entities. + */ +export const composedChange = ( + field: string, + entity: { + readonly before: JsonObject; + readonly after: JsonObject; + readonly sent: JsonObject; + readonly idField: string; + readonly others: readonly JsonObject[]; + }, +): FieldRule | undefined => { + const { before, after, sent, idField } = entity; + const text = after[field]; + if (typeof text !== "string" || typeof before[field] !== "string") + return undefined; + const known = Object.keys(after).filter( + (other) => + other !== field && + (Object.hasOwn(sent, other) + ? same(sent[other], after[other]) + : Object.hasOwn(before, other) && same(before[other], after[other])), + ); + const index = indexed( + known.flatMap((other) => + candidatesOf( + other, + after[other], + other === idField ? 1 : COPY_MIN_LENGTH, + ), + ), + ); + const parts = segmented(text, index, samePart); + return parts === undefined || + composedValue(parts, before) !== before[field] || + heldSpellings(parts, field, entity.others) === undefined + ? undefined + : { field, rule: "composed", parts }; +}; diff --git a/src/domain/analysisView/environmentStateWriteCreate.ts b/src/domain/analysisView/environmentStateWriteCreate.ts index f8f39e7202..78c1f27c4e 100644 --- a/src/domain/analysisView/environmentStateWriteCreate.ts +++ b/src/domain/analysisView/environmentStateWriteCreate.ts @@ -33,6 +33,7 @@ import { import { built, callOf, + clockEvidence, fail, itemReads, refuse, @@ -47,6 +48,7 @@ import { type WriteStep, } from "./environmentStateWriteEffects.js"; import { + createdInteger, explainCreate, insertCandidates, insertIndex, @@ -61,13 +63,20 @@ import { } from "./environmentStateWriteObserve.js"; import { pathValuesOf, + siblingTrimmedClocks, type FieldRule, type WriteCall, } from "./environmentStateWriteRules.js"; import { + fittedClockFormat, + noteFixedWidths, +} from "./environmentStateWriteClockFormats.js"; +import { + containsDelimited, + generatedIntegerDigits, isUuid, + likeGeneratedInteger, nextSequence, - readClock, sequenceNumber, spelledLike, } from "./environmentStateWriteValues.js"; @@ -133,6 +142,47 @@ const createdObservation = ( const fieldsOf = (entities: readonly StateEntity[]): JsonObject[] => entities.map((entity) => entity.fields); +/** Whether a created field holds a generated token or integer that another field holds too, which no rule rebuilds. */ +const linkedGenerated = ( + field: string, + entity: JsonObject, + type: readonly JsonObject[], +): boolean => + createdToken(field, entity, type) === "linked" || + createdInteger(field, entity, type) === "linked"; + +/** Whether a field's rule rebuilds it from `source`, as a template or a composition does. */ +const rebuiltFrom = (rule: FieldRule | undefined, source: string): boolean => + rule?.rule === "template" + ? rule.from === source + : rule?.rule === "composed" && + rule.parts.some((part) => "from" in part && part.from === source); + +/** Whether a unique integer's digits show in another string whose rule does not rebuild them from it, which seeding would leave behind. */ +const strandedInteger = ( + rules: readonly FieldRule[], + entity: JsonObject, +): boolean => + rules.some((rule) => { + const digits = + rule.rule === "unique" + ? generatedIntegerDigits(entity[rule.field]) + : undefined; + return ( + digits !== undefined && + Object.entries(entity).some( + ([other, kept]) => + other !== rule.field && + typeof kept === "string" && + containsDelimited(kept, digits) && + !rebuiltFrom( + rules.find((candidate) => candidate.field === other), + rule.field, + ), + ) + ); + }); + /** Build a create plan from its exemplar's observation. */ const buildCreate = ( context: EffectContext, @@ -163,8 +213,9 @@ const buildCreate = ( Object.keys(evidence.observed.object).some( (field) => !explainedFields.has(field) && - createdToken(field, evidence.observed.object, held) === "linked", - ) + linkedGenerated(field, evidence.observed.object, held), + ) || + strandedInteger(explained.rules, evidence.observed.object) ) { refuse(draft, step, "unexplained-change"); return false; @@ -221,6 +272,7 @@ const buildCreate = ( insert: draft.candidates[0]!, }); draft.observed = evidence.observed.object; + for (const entity of held) noteFixedWidths(draft.fixedWidth, entity); return true; }; @@ -269,21 +321,37 @@ const followsSequences = ( ); }); -/** Whether a later call's generated values take the forms serving generates them in: clocks in the learned format, unique values as the exemplar's kind of token, UUID identities as UUIDs, and sequences in order. */ -const followsGenerated = ( +/** Whether a later create preserves generation shapes and its derived fields reproduce the captured observation. */ +const followsCreateRules = ( context: EffectContext, draft: WriteDraft, - created: Pick, + created: Pick & { + readonly observed: JsonObject; + }, ): boolean => draft.plan.rules.every((rule) => { const value = created.fields[rule.field]; + const exemplar = draft.observed?.[rule.field]; + if (rule.rule === "composed") + return ( + !Object.hasOwn(created.observed, rule.field) || + value === created.observed[rule.field] + ); if (rule.rule === "clock") return ( - JSON.stringify(readClock(value)?.format) === JSON.stringify(rule.format) + fittedClockFormat( + rule.format, + value, + clockEvidence(draft, rule.field), + ) !== undefined ); if (rule.rule === "unique") return ( - (isUuid(value) || isOpaqueToken(value, draft.observed?.[rule.field])) && + (isUuid(exemplar) + ? isUuid(value) + : generatedIntegerDigits(exemplar) !== undefined + ? likeGeneratedInteger(value, exemplar) + : isOpaqueToken(value, exemplar)) && !context.state .ofType(created.type) .some((held) => held.fields[rule.field] === value) @@ -293,6 +361,28 @@ const followsGenerated = ( ); }) && followsSequences(context, draft, created); +/** Widen the plan's clock formats to the trimmed spelling a later call showed, and its other clocks with them unless a value the plan saw shows them at a fixed width. */ +const fitClocks = (draft: WriteDraft, fields: JsonObject): void => { + const fitted = draft.plan.rules.map((rule) => { + if (rule.rule !== "clock") return rule; + const format = fittedClockFormat( + rule.format, + fields[rule.field], + clockEvidence(draft, rule.field), + ); + return format === undefined || format === rule.format + ? rule + : { ...rule, format }; + }); + draft.plan.rules = siblingTrimmedClocks( + fitted, + draft.observed, + fields, + Object.fromEntries(draft.fixedWidth), + ); + noteFixedWidths(draft.fixedWidth, fields); +}; + /** Record the values a create's sequences took, which the next call follows. */ const takeSequences = ( draft: WriteDraft, @@ -458,15 +548,16 @@ export const applyCreate = ( const later = built(draft); if ( later && - draft.plan.rules.some( + (draft.plan.rules.some( (rule) => rule.rule === "unique" && - createdToken( + linkedGenerated( rule.field, observed.object, fieldsOf(context.state.ofType(type)), - ) === "linked", - ) + ), + ) || + strandedInteger(draft.plan.rules, observed.object)) ) { refuse(draft, step, "unexplained-change"); return recorded(); @@ -500,16 +591,18 @@ export const applyCreate = ( id === undefined || context.state.entity(type, collection, id) !== undefined || (later && - !followsGenerated(context, draft, { + !followsCreateRules(context, draft, { type, collection, fields: fields.value, + observed: observed.object, })) ) { refuse(draft, step, "unexplained-change"); return recorded(); } takeSequences(draft, { collection, fields: fields.value }); + fitClocks(draft, fields.value); // The call's own values follow the forms serving generates; the state holds a nested clock as the read-back restamped it, as an update's. const stored = restampedFields( searchFrom(context, step), diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index 47acc73df6..088f9d4dfb 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -48,12 +48,18 @@ import { } from "./environmentStateWriteObserve.js"; import { plannedBody, restampedFields } from "./environmentStateWriteStamps.js"; import { readClock } from "./environmentStateWriteValues.js"; +import { + fittedClockFormat, + matchesClockFormat, + noteFixedWidths, +} from "./environmentStateWriteClockFormats.js"; import { actionItemPath, observationNeeds, } from "./environmentStateWriteAddress.js"; import { pathValuesOf, + siblingTrimmedClocks, type FieldRule, type InsertRule, type StateWritePlan, @@ -84,8 +90,17 @@ export interface WriteDraft { readonly steady: Set; /** Unsent clocks a call left as they were in its own second, which it cannot show it stamps. */ readonly unstamped: Set; + /** A value a held entity or an accepted call showed each clock field at a fixed width in, which keeps it untrimmed. */ + readonly fixedWidth: Map; } +/** The value a draft fits a clock field against: one showing it at a fixed width, else the exemplar's. */ +export const clockEvidence = ( + draft: WriteDraft, + field: string, +): JsonValue | undefined => + draft.fixedWidth.get(field) ?? draft.observed?.[field]; + /** A write observation plus the fields its selected read view actually covers, if any. */ type UpdateObservation = Observed & { readonly readFields?: readonly string[]; @@ -435,6 +450,8 @@ const buildUpdate = ( rules: evidence.explained.rules, }); draft.observed = evidence.after.object; + for (const entity of context.state.ofType(draft.plan.target!)) + noteFixedWidths(draft.fixedWidth, entity.fields); }; /** @@ -509,19 +526,25 @@ export const updateEntity = ( } const unknown = context.unknown(type, collection, id); const call = { ...callOf(step, body), body }; - const explained = explainUpdate({ - before: entity.fields, - after: after.object, - call, - method: step.tool.method, - idField, - id, - startMs: startOf(context, step), - ...(after.readFields === undefined ? {} : { readFields: after.readFields }), - others: otherFields(context.state.ofType(type), { collection, id }), - shown: shownBefore(context.viewOf, entity, after.ordinal), - action: needs.action, - }); + const { explained, rules } = fitUpdateClocks( + draft, + after.object, + explainUpdate({ + before: entity.fields, + after: after.object, + call, + method: step.tool.method, + idField, + id, + startMs: startOf(context, step), + ...(after.readFields === undefined + ? {} + : { readFields: after.readFields }), + others: otherFields(context.state.ofType(type), { collection, id }), + shown: shownBefore(context.viewOf, entity, after.ordinal), + action: needs.action, + }), + ); if (explained.unexplained.length > 0 && !unknown) { refuse(draft, step, "unexplained-change"); return recorded(); @@ -529,11 +552,19 @@ export const updateEntity = ( // Each call must agree with the plan; one on an unknown entity cannot be checked, so it adds nothing to the plan. if ( built(draft) && - (unknown || contradictsPlan(draft, entity.fields, after.object, explained)) + (unknown || + contradictsPlan( + { ...draft, plan: { ...draft.plan, rules } }, + entity.fields, + after.object, + explained, + )) ) { if (!unknown) refuse(draft, step, "unexplained-change"); return recorded(); } + draft.plan.rules = rules; + noteFixedWidths(draft.fixedWidth, after.object); buildUpdate(context, draft, step, { after, explained, idField }); // Serving reads the plan's observation, so the gate does too; generated values are this call's. const written = updatedFields( @@ -586,6 +617,54 @@ export const updateEntity = ( }; }; +/** A later update's clocks in the plan's format, and the plan's rules widened to the trimmed spelling the call showed when one covers both, its siblings with it wherever the call's values fit; the plan adopts them only once the call is checked. */ +const fitUpdateClocks = ( + draft: WriteDraft, + after: JsonObject, + explained: ReturnType, +): { + readonly explained: ReturnType; + readonly rules: FieldRule[]; +} => { + if (!built(draft)) return { explained, rules: draft.plan.rules }; + const rules = draft.plan.rules.map((rule) => { + const later = explained.rules.find((other) => other.field === rule.field); + if ( + rule.rule !== "clock" || + later?.rule !== "clock" || + JSON.stringify(later.format) === JSON.stringify(rule.format) + ) + return rule; + const format = fittedClockFormat( + rule.format, + after[rule.field], + clockEvidence(draft, rule.field), + ); + return format === undefined ? rule : { ...rule, format }; + }); + const fitted = siblingTrimmedClocks( + rules, + draft.observed, + after, + Object.fromEntries(draft.fixedWidth), + ); + const planned = new Map(fitted.map((rule) => [rule.field, rule])); + return { + explained: { + ...explained, + rules: explained.rules.map((rule) => { + const held = planned.get(rule.field); + return rule.rule === "clock" && + held?.rule === "clock" && + matchesClockFormat(held.format, readClock(after[rule.field])) + ? { ...rule, format: held.format } + : rule; + }), + }, + rules: fitted, + }; +}; + /** Serving regenerates a learned clock on every update, so recorded calls must advance it. */ const contradictsGeneratedClock = ( before: JsonObject, diff --git a/src/domain/analysisView/environmentStateWriteExplain.test.ts b/src/domain/analysisView/environmentStateWriteExplain.test.ts index a7cb1bd845..101a18087d 100644 --- a/src/domain/analysisView/environmentStateWriteExplain.test.ts +++ b/src/domain/analysisView/environmentStateWriteExplain.test.ts @@ -1109,3 +1109,89 @@ it("accepts a copied member only with its recorded value when a template keeps i sent: "group-alpha-2", }); }); + +it.each<[string, JsonValue, readonly JsonValue[], boolean]>([ + [ + "a time-based id no held entity takes", + 1656166527623168, + [1655476044521472, 1655476044521999], + true, + ], + [ + "one a held entity already holds", + 1655476044521472, + [1655476044521472], + false, + ], + [ + "one among held ids of another length", + 1656166527623168, + [16554760445214], + false, + ], + [ + "one among held ids spelled as text", + 1656166527623168, + ["1655476044521472"], + false, + ], + [ + "one among held entities sharing an id", + 1656166527623168, + [1655476044521472, 1655476044521472], + false, + ], + ["one with no held entity to compare", 1656166527623168, [], false], + ["a short number", 123456789, [987654321], false], +])("learns %s as generated", (_name, value, held, unique) => { + const result = explainCreate( + created( + { id: 7, ref: value }, + {}, + { type: held.map((ref, index) => ({ id: index + 1, ref })) }, + ), + ); + expect(ruleFor(result.rules, "ref")).toEqual( + unique ? { field: "ref", rule: "unique" } : undefined, + ); +}); + +it("trims a created clock's fraction, and its sibling clock's, when held entities spell the field trimmed", () => { + const at = new Date(START).toISOString().replace(".000Z", ""); + const result = explainCreate( + created( + { + id: 7, + createdAt: `${at}.123456789Z`, + updatedAt: `${at}.123456791Z`, + }, + {}, + { + type: [ + { + id: 1, + createdAt: "2026-01-01T00:00:00.12345678Z", + updatedAt: "2026-01-01T00:00:00.123456789Z", + }, + ], + }, + ), + ); + const trimmed = { + kind: "iso", + separator: "T", + fraction: 9, + zone: "Z", + trimmed: true, + }; + expect(ruleFor(result.rules, "createdAt")).toEqual({ + field: "createdAt", + rule: "clock", + format: trimmed, + }); + expect(ruleFor(result.rules, "updatedAt")).toEqual({ + field: "updatedAt", + rule: "clock", + format: trimmed, + }); +}); diff --git a/src/domain/analysisView/environmentStateWriteExplain.ts b/src/domain/analysisView/environmentStateWriteExplain.ts index 26ad07dcea..9bcbdbb3b9 100644 --- a/src/domain/analysisView/environmentStateWriteExplain.ts +++ b/src/domain/analysisView/environmentStateWriteExplain.ts @@ -9,21 +9,30 @@ import { type JsonObject, type StateEntity, } from "./environmentStateRules.js"; -import type { - FieldRule, - InsertRule, - WriteBody, - WriteCall, - WriteResponse, +import { + siblingTrimmedClocks, + type FieldRule, + type InsertRule, + type WriteBody, + type WriteCall, + type WriteResponse, } from "./environmentStateWriteRules.js"; import { containsDelimited, + containsNumber, + generatedIntegerDigits, integerOf, isUuid, readClock, sequenceNumber, slugOf, + type ClockFormat, } from "./environmentStateWriteValues.js"; +import { createdComposer } from "./environmentStateWriteComposed.js"; +import { + fittedClockFormat, + showsFixedWidth, +} from "./environmentStateWriteClockFormats.js"; import { COPY_MIN_LENGTH, instantMembers, @@ -157,6 +166,64 @@ const requestRule = ( : { field, rule: "slug", from: member }; }; +/** A clock's format, trimmed when the type's values of the field show its fraction trimmed and none shows a fixed width. */ +const learnedClockFormat = ( + format: ClockFormat, + field: string, + value: JsonValue, + type: readonly JsonObject[], +): ClockFormat => { + let learned = format; + for (const member of type) { + const fitted = fittedClockFormat(learned, member[field], value); + if (fitted?.kind === "iso" && fitted.trimmed === true) learned = fitted; + } + return type.some((member) => showsFixedWidth(format, member[field])) + ? format + : learned; +}; + +/** A created large integer's proof, as a time-based id's: no entity of its type holds it and each holds a distinct one of the same length and kind; linked when another field holds it too, as a number at any depth or delimited in a nested string, which seeding cannot keep. */ +export const createdInteger = ( + field: string, + entity: JsonObject, + type: readonly JsonObject[], +): "independent" | "linked" | undefined => { + const value = entity[field]; + const digits = generatedIntegerDigits(value); + if ( + digits === undefined || + readClock(value) !== undefined || + type.length === 0 + ) + return undefined; + const held = new Set([digits]); + const distinct = type.every((member) => { + const other = generatedIntegerDigits(member[field]); + if ( + other === undefined || + typeof member[field] !== typeof value || + other.length !== digits.length || + held.has(other) + ) + return false; + held.add(other); + return true; + }); + if (!distinct) return undefined; + return Object.entries(entity).some( + ([other, kept]) => + other !== field && + (kept === value || + containsNumber(kept, digits) || + (typeof kept === "object" && + kept !== null && + containsDelimited(kept, digits))), + ) + ? "linked" + : "independent"; +}; + /** The rule a created field's generated value follows, if it follows one. */ const generatedRule = ( evidence: CreateEvidence, @@ -166,7 +233,11 @@ const generatedRule = ( if (field === evidence.idField) return identityRule(field, value); const clock = readClock(value); if (clock !== undefined && isClockNear(value, evidence.startMs)) - return { field, rule: "clock", format: clock.format }; + return { + field, + rule: "clock", + format: learnedClockFormat(clock.format, field, value, evidence.type), + }; const integer = sequenceNumber(value); if (integer !== undefined) { if (nextInteger(evidence.collection, field, value) === integer) @@ -175,9 +246,10 @@ const generatedRule = ( return { field, rule: "sequence", scope: "type" }; } if ( - isUuid(value) && - evidence.type.length > 0 && - evidence.type.every((member) => member[field] !== value) + (isUuid(value) && + evidence.type.length > 0 && + evidence.type.every((member) => member[field] !== value)) || + createdInteger(field, evidence.entity, evidence.type) === "independent" ) return { field, rule: "unique" }; if (createdToken(field, evidence.entity, evidence.type) === "independent") @@ -208,7 +280,7 @@ export const explainCreate = ( readonly body: WriteBody; readonly tied: boolean; } => { - const rules: FieldRule[] = []; + const ruled: FieldRule[] = []; const unexplained: string[] = []; const index = indexBody(evidence.call.body); for (const [field, value] of Object.entries(evidence.entity)) { @@ -216,8 +288,9 @@ export const explainCreate = ( requestRule(evidence, index, field, value) ?? generatedRule(evidence, field, value); if (rule === undefined) unexplained.push(field); - else rules.push(rule); + else ruled.push(rule); } + const rules = siblingTrimmedClocks(ruled, evidence.entity, ...evidence.type); // Generated values first, then copies: an echoed value may recur in unrelated text, a copied one is the call's own unless also echoed. const echoed = new Set( rules.flatMap((rule) => @@ -233,8 +306,10 @@ export const explainCreate = ( ), ]; const constants: string[] = []; + const composed = createdComposer(evidence.entity, rules, evidence.type); for (const field of unexplained) { - const template = templateRule(evidence.entity, field, sources); + const template = + templateRule(evidence.entity, field, sources) ?? composed(field); if (template === undefined) constants.push(field); else rules.push(template); } @@ -244,6 +319,12 @@ export const explainCreate = ( const templates = rules.flatMap((rule) => rule.rule === "template" ? [rule.field] : [], ); + // A composed string rebuilds the values it names, so only its literal text can tie it. + const literals = rules.flatMap((rule) => + rule.rule === "composed" + ? rule.parts.flatMap((part) => ("text" in part ? [part.text] : [])) + : [], + ); // A member whose value a template cannot rebuild, or that a constant derives from, is accepted only with its recorded value. const texts = [...constants, ...templates]; const stranded = strandedCopies(rules, evidence.entity, texts); @@ -277,9 +358,10 @@ export const explainCreate = ( Object.keys(evidence.call.body), ), // A template rebuilds only its generated value, so other text it keeps ties it like a constant. - tied: [...constants, ...templates].some((field) => - tokens.some((token) => containsDelimited(evidence.entity[field]!, token)), - ), + tied: [ + ...[...constants, ...templates].map((field) => evidence.entity[field]!), + ...literals, + ].some((text) => tokens.some((token) => containsDelimited(text, token))), }; }; diff --git a/src/domain/analysisView/environmentStateWriteRules.test.ts b/src/domain/analysisView/environmentStateWriteRules.test.ts new file mode 100644 index 0000000000..e0e3d7c2ff --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteRules.test.ts @@ -0,0 +1,52 @@ +import { expect, it } from "vitest"; +import { + siblingTrimmedClocks, + type FieldRule, +} from "./environmentStateWriteRules.js"; + +it.each([false, true])( + "propagates the widest trimmed peer in each separator/zone group (reversed: %s)", + (reverse) => { + const clock = ( + field: string, + separator: "T" | " ", + zone: string, + fraction: number, + trimmed = false, + ): FieldRule => ({ + field, + rule: "clock", + format: { + kind: "iso", + separator, + zone, + fraction, + ...(trimmed ? { trimmed: true } : {}), + }, + }); + const rules = [ + clock("utc", "T", "Z", 9, true), + clock("shortOffset", "T", "+01:00", 6, true), + clock("wideOffset", "T", "+01:00", 9, true), + clock("offsetSibling", "T", "+01:00", 8), + clock("spaced", " ", "+01:00", 9, true), + clock("spacedSibling", " ", "+01:00", 8), + clock("fixedSibling", "T", "+01:00", 9), + clock("otherZone", "T", "-01:00", 8), + ]; + const fitted = siblingTrimmedClocks(reverse ? rules.toReversed() : rules, { + offsetSibling: "2026-03-04T05:06:07.12345678+01:00", + spacedSibling: "2026-03-04 05:06:07.12345678+01:00", + fixedSibling: "2026-03-04T05:06:07.123456780+01:00", + otherZone: "2026-03-04T05:06:07.12345678-01:00", + }); + expect(fitted).toContainEqual( + clock("offsetSibling", "T", "+01:00", 9, true), + ); + expect(fitted).toContainEqual( + clock("spacedSibling", " ", "+01:00", 9, true), + ); + expect(fitted).toContainEqual(clock("fixedSibling", "T", "+01:00", 9)); + expect(fitted).toContainEqual(clock("otherZone", "T", "-01:00", 8)); + }, +); diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts index 4b70317bec..badc28748f 100644 --- a/src/domain/analysisView/environmentStateWriteRules.ts +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -8,6 +8,11 @@ import { type JsonObject, } from "./environmentStateRules.js"; import type { ClockFormat } from "./environmentStateWriteValues.js"; +import { + showsFixedWidth, + trimmedLike, + type ClockPattern, +} from "./environmentStateWriteClockFormats.js"; /** What a write tool does to the state, by method and the target its path names. */ export type WriteEffect = "create" | "update" | "delete" | "document" | "none"; @@ -70,6 +75,12 @@ export type FieldRule = | { readonly field: string; readonly rule: "path"; readonly from: string } | { readonly field: string; readonly rule: "slug"; readonly from: string } | { readonly field: string; readonly rule: "template"; readonly from: string } + /** A string rebuilt from literal text and other written fields, in order. */ + | { + readonly field: string; + readonly rule: "composed"; + readonly parts: ComposedPart[]; + } | { readonly field: string; readonly rule: "increment" } /** An update stores `value` whenever the call sends the member, as the call that showed it did. */ | { @@ -98,6 +109,50 @@ export type FieldRule = readonly value: JsonValue; }; +/** Clock rules spelled as a trimmed sibling is, since an application spells its clocks alike: fields whose `values`, and every `later` call's, show no fixed width. */ +export const siblingTrimmedClocks = ( + rules: readonly FieldRule[], + values: Readonly> | undefined, + ...later: readonly Readonly>[] +): FieldRule[] => { + const trimmed = new Map>(); + for (const rule of rules) { + if ( + rule.rule !== "clock" || + rule.format.kind !== "iso" || + rule.format.trimmed !== true + ) + continue; + const format = rule.format; + const group = `${format.separator}${format.zone}`; + const widest = trimmed.get(group); + if (widest === undefined || format.fraction > widest.fraction) + trimmed.set(group, format); + } + if (trimmed.size === 0) return [...rules]; + return rules.map((rule) => { + if (rule.rule !== "clock" || rule.format.kind !== "iso") return rule; + const peer = trimmed.get(`${rule.format.separator}${rule.format.zone}`); + if (peer === undefined) return rule; + const format = later.some((fields) => + showsFixedWidth(rule.format, fields[rule.field]), + ) + ? rule.format + : trimmedLike(rule.format, peer, values?.[rule.field]); + return format === rule.format ? rule : { ...rule, format }; + }); +}; + +/** One part of a composed string: literal text, or another written field as written, slugged, or as a clock's calendar digits. */ +export type ComposedPart = + | { readonly text: string } + | { readonly from: string; readonly as: "value" | "slug" } + | { + readonly from: string; + readonly as: "clock"; + readonly pattern: ClockPattern; + }; + /** Where a create places its entity in its collection's order. */ export type InsertRule = | { readonly rule: "first" | "last" } diff --git a/src/domain/analysisView/environmentStateWriteValues.test.ts b/src/domain/analysisView/environmentStateWriteValues.test.ts index fc8d3103e4..a719b46a30 100644 --- a/src/domain/analysisView/environmentStateWriteValues.test.ts +++ b/src/domain/analysisView/environmentStateWriteValues.test.ts @@ -4,11 +4,13 @@ import { containsDelimited, delimitedOffsets, formatClock, + generatedIntegerDigits, integerOf, isUuid, readClock, replaceDelimited, respellClock, + seededInteger, seededToken, seededUuid, sequenceNumber, @@ -69,10 +71,24 @@ it.each<[string, number, ClockFormat, number?]>([ ])( "reads the ISO clock %s with its spelling", (text, ms, format, subMillisecond = 0) => { - expect(readClock(text)).toEqual({ ms, subMillisecond, format }); + expect(readClock(text)).toEqual({ + ms, + subMillisecond, + format, + fixedWidth: false, + }); }, ); +it.each<[string, boolean]>([ + ["2026-03-04T05:06:07.120Z", true], + ["2026-03-04T05:06:07.000Z", true], + ["2026-03-04T05:06:07.12Z", false], + ["2026-03-04T05:06:07Z", false], +])("reads whether the fraction of %s shows a fixed width", (text, fixed) => { + expect(readClock(text)?.fixedWidth).toBe(fixed); +}); + it.each([ "0001-02-29T00:00:00Z", "0099-02-29T00:00:00Z", @@ -153,11 +169,17 @@ it.each<[JsonValue | undefined, ReturnType]>([ ms: WHOLE_SECOND, subMillisecond: 0, format: { kind: "epoch", unit: "s" }, + fixedWidth: true, }, ], [ WITH_MS, - { ms: WITH_MS, subMillisecond: 0, format: { kind: "epoch", unit: "ms" } }, + { + ms: WITH_MS, + subMillisecond: 0, + format: { kind: "epoch", unit: "ms" }, + fixedWidth: true, + }, ], [999_999_999, undefined], [99_999_999_999, undefined], @@ -413,3 +435,42 @@ it("respells a clock at another offset with its own fraction digits, across a da "2026-10-21T17:06:13.250+00:00", ); }); + +const TRIMMED_9: ClockFormat = { + kind: "iso", + separator: "T", + fraction: 9, + zone: "Z", + trimmed: true, +}; + +it("spells a trimmed format without trailing fraction zeros, and without the point when none remain", () => { + expect(formatClock(WITH_MS, TRIMMED_9)).toBe("2026-03-04T05:06:07.123Z"); + expect(formatClock(WHOLE_SECOND, TRIMMED_9)).toBe("2026-03-04T05:06:07Z"); + expect(respellClock("2026-03-04T05:06:07.120000Z", TRIMMED_9)).toBe( + "2026-03-04T05:06:07.12Z", + ); +}); + +it("seeds integers as long as the recorded one, keeping its JSON kind and a safe number", () => { + const number = seededInteger("seed", 1656166527623168); + expect(typeof number).toBe("number"); + expect(Number.isSafeInteger(number)).toBe(true); + expect(String(number)).toMatch(/^[1-8]\d{15}$/u); + expect(seededInteger("seed", 1656166527623168)).toBe(number); + expect(seededInteger("other", 1656166527623168)).not.toBe(number); + expect(seededInteger("seed", "1234567890123456789")).toMatch( + /^[1-9]\d{18}$/u, + ); +}); + +it.each<[JsonValue, string | undefined]>([ + [1656166527623168, "1656166527623168"], + ["1234567890123456789", "1234567890123456789"], + [123456789, undefined], + ["0123456789012", undefined], + [-1656166527623168, undefined], + [1.5e15 + 0.5, undefined], +])("reads %j as a generated integer's digits", (value, expected) => { + expect(generatedIntegerDigits(value)).toBe(expected); +}); diff --git a/src/domain/analysisView/environmentStateWriteValues.ts b/src/domain/analysisView/environmentStateWriteValues.ts index 46b6afe0e0..8ab0d19bcb 100644 --- a/src/domain/analysisView/environmentStateWriteValues.ts +++ b/src/domain/analysisView/environmentStateWriteValues.ts @@ -9,14 +9,18 @@ export type ClockFormat = readonly separator: "T" | " "; readonly fraction: number; readonly zone: string; + /** Fraction digits end at the last non-zero one, and the point goes with them, as Go spells nanoseconds; `fraction` is the most shown. */ + readonly trimmed?: true; } | { readonly kind: "epoch"; readonly unit: "s" | "ms" }; -interface ClockReading { +export interface ClockReading { readonly ms: number; /** Nanoseconds after the floored millisecond, so distinct recorded instants stay distinct. */ readonly subMillisecond: number; readonly format: ClockFormat; + /** The fraction ends in a zero, which a trimmed spelling never shows, so its width is fixed. */ + readonly fixedWidth: boolean; } const MINUTE_MS = 60_000; @@ -37,7 +41,8 @@ const zoneMinutes = (zone: string): number | undefined => { export const isClockZone = (zone: string): boolean => zoneMinutes(zone) !== undefined; -const zoneMinutesForFormat = (zone: string): number => { +/** Minutes east of UTC a stored format's zone names. */ +export const zoneMinutesForFormat = (zone: string): number => { const minutes = zoneMinutes(zone); if (minutes === undefined) throw new TypeError(`Invalid clock zone in a derived format: ${zone}`); @@ -64,6 +69,8 @@ const readIsoClock = (value: string): ClockReading | undefined => { fraction: spelling.fractionDigits, zone: spelling.zone, }, + // The instant's fraction drops trailing zeros, so a longer spelling ended in one. + fixedWidth: spelling.fractionDigits > instant.fraction.length, }; }; @@ -79,12 +86,14 @@ export const readClock = ( ms: value * 1000, subMillisecond: 0, format: { kind: "epoch", unit: "s" }, + fixedWidth: true, }; if (value >= 1e12 && value < 1e13) return { ms: value, subMillisecond: 0, format: { kind: "epoch", unit: "ms" }, + fixedWidth: true, }; return undefined; }; @@ -116,7 +125,12 @@ export const formatClock = ( 0, format.fraction, ); - return isoText(at, format.separator, digits, format.zone); + return isoText( + at, + format.separator, + format.trimmed === true ? digits.replace(/0+$/u, "") : digits, + format.zone, + ); }; /** @@ -147,7 +161,9 @@ export const respellClock = ( isoText( at, format.separator, - digits.padEnd(format.fraction, "0"), + format.trimmed === true + ? digits.replace(/0+$/u, "") + : digits.padEnd(format.fraction, "0"), format.zone, ) ?? value ); @@ -158,6 +174,39 @@ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; export const isUuid = (value: JsonValue | undefined): value is string => typeof value === "string" && UUID.test(value); +/** Digits a generated integer shows at least, as a time-based id does; counts and codes are shorter. */ +const GENERATED_INTEGER_DIGITS = 10; + +/** A large integer's digits, from a safe JSON number or a string of digits without a leading zero. */ +export const generatedIntegerDigits = ( + value: JsonValue | undefined, +): string | undefined => { + const digits = + typeof value === "number" && Number.isSafeInteger(value) && value > 0 + ? String(value) + : typeof value === "string" && /^[1-9]\d*$/u.test(value) + ? value + : undefined; + return digits !== undefined && + digits.length >= GENERATED_INTEGER_DIGITS && + digits.length <= 20 + ? digits + : undefined; +}; + +/** Whether a generated integer retains the exemplar's digit count and JSON kind. */ +export const likeGeneratedInteger = ( + value: JsonValue | undefined, + exemplar: JsonValue | undefined, +): boolean => { + const digits = generatedIntegerDigits(value); + return ( + digits !== undefined && + typeof value === typeof exemplar && + digits.length === generatedIntegerDigits(exemplar)?.length + ); +}; + /** An integer value as a number, from a JSON number or a string of digits. */ export const integerOf = (value: JsonValue | undefined): number | undefined => typeof value === "number" && Number.isSafeInteger(value) @@ -260,6 +309,14 @@ const nextCharacter = (text: string, at: number): string | undefined => { return point === undefined ? undefined : String.fromCodePoint(point); }; +/** Whether the character before UTF-16 offset `at` is a letter or digit, joining text that starts there to its left. */ +export const joinsBefore = (text: string, at: number): boolean => + isAlphanumeric(previousCharacter(text, at)); + +/** Whether the character at UTF-16 offset `at` is a letter or digit, joining text that ends there to its right. */ +export const joinsAfter = (text: string, at: number): boolean => + isAlphanumeric(nextCharacter(text, at)); + /** Start offsets of `token` in `text` where neither neighbour is a letter or digit. */ export const delimitedOffsets = (text: string, token: string): number[] => { const offsets: number[] = []; @@ -267,8 +324,7 @@ export const delimitedOffsets = (text: string, token: string): number[] => { let at = text.indexOf(token); while (at >= 0) { const delimited = - !isAlphanumeric(previousCharacter(text, at)) && - !isAlphanumeric(nextCharacter(text, at + token.length)); + !joinsBefore(text, at) && !joinsAfter(text, at + token.length); if (delimited) offsets.push(at); // A rejected match may overlap a delimited one, so the search moves on by one character. at = text.indexOf(token, at + (delimited ? token.length : 1)); @@ -305,6 +361,19 @@ export const containsDelimited = (value: JsonValue, token: string): boolean => { return false; }; +/** Whether any number inside a value spells `digits`. */ +export const containsNumber = (value: JsonValue, digits: string): boolean => { + const pending: JsonValue[] = [value]; + for (let next = pending.pop(); next !== undefined; next = pending.pop()) { + if (typeof next === "number") { + if (String(next) === digits) return true; + } else if (typeof next === "object" && next !== null) + for (const child of Array.isArray(next) ? next : Object.values(next)) + pending.push(child); + } + return false; +}; + /** SHA-256 bytes of a seed, extended by counter blocks to the requested length. */ const seededBytes = (seed: string, length: number): Uint8Array => { const bytes = new Uint8Array(length); @@ -359,6 +428,20 @@ export const seededLike = ( ? seededUuid(seed, String(exemplar)) : seededToken(seed, String(exemplar)); +/** A seeded integer with as many digits as the recorded one and none leading zero, as a number when it is one and stays safe. */ +export const seededInteger = (seed: string, exemplar: JsonValue): JsonValue => { + const length = String(exemplar).length; + const bytes = seededBytes(seed, length); + // A number keeps under the largest safe integer, which a sixteen-digit one starting with 9 can pass. + const leading = typeof exemplar === "number" && length >= 16 ? 8 : 9; + const digits = [...bytes] + .map((byte, index) => + String(index === 0 ? 1 + (byte % leading) : byte % 10), + ) + .join(""); + return typeof exemplar === "number" ? Number(digits) : digits; +}; + /** A token as long as the recorded one, from its alphabet, keeping its other characters in place. */ export const seededToken = (seed: string, exemplar: string): string => { const alphabet = tokenAlphabet(exemplar); diff --git a/tests/boundary/cli/environmentCompareCli.test.ts b/tests/boundary/cli/environmentCompareCli.test.ts index a965ca12d3..b7776f3ac9 100644 --- a/tests/boundary/cli/environmentCompareCli.test.ts +++ b/tests/boundary/cli/environmentCompareCli.test.ts @@ -10,6 +10,11 @@ import { type TimedCaptureRow, } from "../../fixtures/analysisView.js"; import { environmentStateServeFixture } from "../../fixtures/environmentServe.js"; +import { + BOOKMARK_ROWS, + bookmark, + TAG_ROWS, +} from "../../fixtures/environmentStateDerivedValues.js"; import { BOOK_NOT_FOUND, BOOK_ROWS, @@ -349,6 +354,91 @@ describe("rea environment-compare value context", () => { }); describe("rea environment-compare recorded clock spelling", () => { + for (const scenario of [ + { + name: "variable fractional widths under a confirmed trimmed write plan", + rows: TAG_ROWS, + probe: [ + [ + "POST", + "/api/tags", + { + id: numberedUuid(20), + name: "Spare", + createdAt: "2026-10-20T10:00:00.12345678Z", + updatedAt: "2026-10-20T10:00:00.12345678Z", + }, + 201, + { name: "Spare" }, + ], + ] as const, + generated: 0, + }, + { + name: "a composed bookmark URL through its creation clock", + rows: BOOKMARK_ROWS, + probe: [ + [ + "POST", + "/api/bookmarks", + bookmark( + 20, + "https://pier.example.test/inspection", + "Pier", + "2026-10-20T10:00:00Z", + ), + 201, + { url: "https://pier.example.test/inspection", title: "Pier" }, + ], + ] as const, + generated: 1, + }, + ]) { + cliTest(`accepts ${scenario.name}`, async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-environment-derived-compare-", + ); + const capture = timedCaptureEvidence(scenario.rows); + const fixture = environmentStateServeFixture(capture); + const probe = timedCaptureEvidence(scenario.probe); + const originals = new Map(); + for (const [name, value] of [ + ["manifest.json", fixture.manifest], + ["capture.json", capture], + ["probe.json", probe], + ] as const) { + const path = join(root, name); + const text = JSON.stringify(value); + originals.set(path, text); + await writeFile(path, text); + } + const result = await cli.run({ + arguments: [ + "environment-compare", + join(root, "manifest.json"), + "--evidence", + join(root, "capture.json"), + "--probe", + join(root, "probe.json"), + "--json", + ], + }); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.json).toMatchObject({ + status: "complete", + summary: { probes: 1, steps: 1, equal: 1, agreement: 1 }, + probes: [ + { + normalised: { clocks: 2, generated: scenario.generated }, + results: [{ method: "POST", equal: true, differences: [] }], + }, + ], + }); + for (const [path, text] of originals) + expect(await readFile(path, "utf8")).toBe(text); + }); + } + cliTest( "replays compact-offset submillisecond probe timestamps in exact order", async ({ cli }) => { diff --git a/tests/fixtures/environmentStateDerivedValues.ts b/tests/fixtures/environmentStateDerivedValues.ts new file mode 100644 index 0000000000..dd18c8fab8 --- /dev/null +++ b/tests/fixtures/environmentStateDerivedValues.ts @@ -0,0 +1,484 @@ +import type { JsonValue } from "../../src/domain/jsonValue.js"; +import type { TimedCaptureRow } from "./analysisView.js"; +import { bookClock, numberedUuid } from "./environmentStateWrites.js"; + +type Entity = { [field: string]: JsonValue }; + +/** The instant `seconds` after the first exchange with nanosecond digits, trailing zeros trimmed as Go spells them. */ +export const goClock = (seconds: number, nanos: string): string => + bookClock(seconds).replace("Z", `.${nanos}`.replace(/\.?0+$/u, "") + "Z"); + +const tag = (n: number, name: string, created: string, updated: string) => ({ + id: numberedUuid(n), + name, + createdAt: created, + updatedAt: updated, +}); + +const FIRST_TAG = tag( + 1, + "Fragile", + "2026-10-01T09:00:00.123456789Z", + "2026-10-01T09:00:00.123456791Z", +); +const SECOND_TAG = tag( + 2, + "Outdoor", + "2026-10-01T10:00:00.223456789Z", + "2026-10-01T10:00:00.223456791Z", +); +/** The first created tag's clocks show nine digits; the second's update clock ends in a zero Go drops. */ +const HEAVY_TAG = tag( + 3, + "Heavy", + goClock(3, "374763655"), + goClock(3, "374763767"), +); +const BORROWED_TAG = tag( + 4, + "Borrowed", + goClock(5, "388122047"), + goClock(5, "388122180"), +); + +/** Two tag creates whose clocks Go spells with trailing fraction zeros trimmed, confirmed by later reads. */ +export const TAG_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/tags", [FIRST_TAG, SECOND_TAG]], + ["GET", `/api/tags/${numberedUuid(1)}`, FIRST_TAG], + ["GET", `/api/tags/${numberedUuid(2)}`, SECOND_TAG], + ["POST", "/api/tags", HEAVY_TAG, 201, { name: "Heavy" }], + ["GET", `/api/tags/${numberedUuid(3)}`, HEAVY_TAG], + ["POST", "/api/tags", BORROWED_TAG, 201, { name: "Borrowed" }], + ["GET", `/api/tags/${numberedUuid(4)}`, BORROWED_TAG], + ["GET", "/api/tags", [FIRST_TAG, SECOND_TAG, HEAVY_TAG, BORROWED_TAG]], +]; + +/** A later tag create whose update clock Go trims while its creation clock keeps a trailing zero. */ +const FIXED_CREATED_TAG = tag( + 4, + "Borrowed", + bookClock(5).replace("Z", ".388122040Z"), + goClock(5, "388122180"), +); + +/** Tag creates whose later call trims one clock and shows its sibling at a fixed width. */ +export const FIXED_SIBLING_TAG_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/tags", [FIRST_TAG, SECOND_TAG]], + ["GET", `/api/tags/${numberedUuid(1)}`, FIRST_TAG], + ["GET", `/api/tags/${numberedUuid(2)}`, SECOND_TAG], + ["POST", "/api/tags", HEAVY_TAG, 201, { name: "Heavy" }], + ["GET", `/api/tags/${numberedUuid(3)}`, HEAVY_TAG], + ["POST", "/api/tags", FIXED_CREATED_TAG, 201, { name: "Borrowed" }], + ["GET", `/api/tags/${numberedUuid(4)}`, FIXED_CREATED_TAG], + ["GET", "/api/tags", [FIRST_TAG, SECOND_TAG, HEAVY_TAG, FIXED_CREATED_TAG]], +]; + +/** A clock's UTC date and time digits, as the Wayback Machine names a snapshot. */ +const archiveDigits = (clock: string): string => + clock.replace(/\D/gu, "").slice(0, 14); + +/** A bookmark whose snapshot address names its creation time and its URL. */ +export const bookmark = ( + id: number, + url: string, + title: string, + added: string, +): Entity => ({ + id, + url, + title, + web_archive_snapshot_url: `https://web.archive.org/web/${archiveDigits(added)}/${url}`, + date_added: added, + date_modified: added, +}); + +const FIRST_BOOKMARK = bookmark( + 1, + "https://ferries.example.test/timetable", + "Ferry timetable", + "2026-10-01T09:00:00Z", +); +const SECOND_BOOKMARK = bookmark( + 2, + "https://harbour.example.test/tides", + "Harbour tides", + "2026-10-01T10:00:00Z", +); +const THIRD_BOOKMARK = bookmark( + 3, + "https://quarry.example.test/ledger", + "Quarry ledger", + bookClock(3), +); +const FOURTH_BOOKMARK = bookmark( + 4, + "https://kitchen.example.test/menus", + "Kitchen menus", + bookClock(5), +); + +/** Two bookmark creates whose snapshot addresses the application composes, confirmed by later reads. */ +export const BOOKMARK_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/bookmarks", [FIRST_BOOKMARK, SECOND_BOOKMARK]], + ["GET", "/api/bookmarks/1", FIRST_BOOKMARK], + ["GET", "/api/bookmarks/2", SECOND_BOOKMARK], + [ + "POST", + "/api/bookmarks", + THIRD_BOOKMARK, + 201, + { url: "https://quarry.example.test/ledger", title: "Quarry ledger" }, + ], + ["GET", "/api/bookmarks/3", THIRD_BOOKMARK], + [ + "POST", + "/api/bookmarks", + FOURTH_BOOKMARK, + 201, + { url: "https://kitchen.example.test/menus", title: "Kitchen menus" }, + ], + ["GET", "/api/bookmarks/4", FOURTH_BOOKMARK], + [ + "GET", + "/api/bookmarks", + [FIRST_BOOKMARK, SECOND_BOOKMARK, THIRD_BOOKMARK, FOURTH_BOOKMARK], + ], +]; + +const slugOf = (title: string): string => + title + .toLowerCase() + .replace(/[^a-z0-9]+/gu, "-") + .replace(/^-|-$/gu, ""); + +/** A folder named by the uid its creator chose, with an address from that uid and its title, and a time-based numeric id. */ +export const folder = ( + uid: string, + title: string, + id: number, + created: string, + updated = created, +): Entity => ({ + id, + uid, + title, + url: `/dashboards/f/${uid}/${slugOf(title)}`, + created, + updated, +}); + +const SEED_FOLDER = folder( + numberedUuid(101), + "Lighthouse Rota", + 1655476044521472, + "2026-10-01T09:00:00Z", +); +const ALPHA = folder( + numberedUuid(102), + "Scratch Alpha Harbor", + 1656166527623168, + bookClock(2), +); +const BRAVO = folder( + numberedUuid(103), + "Scratch Bravo Meadow", + 1656166745726976, + bookClock(4), +); +/** The first created folder after the recorded rename at exchange 6. */ +const RENAMED_ALPHA = folder( + numberedUuid(102), + "Renamed Alpha Lantern", + 1656166527623168, + bookClock(2), + bookClock(6), +); + +/** Two folder creates and a rename, whose addresses follow uid and title, confirmed by later reads. */ +export const FOLDER_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/folders", [SEED_FOLDER]], + ["GET", `/api/folders/${numberedUuid(101)}`, SEED_FOLDER], + [ + "POST", + "/api/folders", + ALPHA, + 200, + { uid: numberedUuid(102), title: "Scratch Alpha Harbor" }, + ], + ["GET", `/api/folders/${numberedUuid(102)}`, ALPHA], + [ + "POST", + "/api/folders", + BRAVO, + 200, + { uid: numberedUuid(103), title: "Scratch Bravo Meadow" }, + ], + ["GET", `/api/folders/${numberedUuid(103)}`, BRAVO], + [ + "PUT", + `/api/folders/${numberedUuid(102)}`, + RENAMED_ALPHA, + 200, + { title: "Renamed Alpha Lantern" }, + ], + ["GET", `/api/folders/${numberedUuid(102)}`, RENAMED_ALPHA], + ["GET", "/api/folders", [SEED_FOLDER, RENAMED_ALPHA, BRAVO]], +]; + +/** Two folder renames, the first leaving its address as it was unless `rebuilt`, which another tool then rebuilds. */ +export const restoredAddressRows = (rebuilt: boolean): TimedCaptureRow[] => { + const held = "2026-10-01T09:00:00Z"; + const alpha = folder( + numberedUuid(102), + "Scratch Alpha Harbor", + 1656166527623168, + held, + ); + const bravo = folder( + numberedUuid(103), + "Scratch Bravo Meadow", + 1656166745726976, + held, + ); + const renamed = folder( + numberedUuid(102), + "Renamed Alpha Lantern", + 1656166527623168, + held, + bookClock(3), + ); + const first = rebuilt ? renamed : { ...renamed, url: alpha.url ?? null }; + const second = folder( + numberedUuid(103), + "Second Bravo Lantern", + 1656166745726976, + held, + bookClock(7), + ); + return [ + ["GET", "/api/folders", [alpha, bravo]], + ["GET", `/api/folders/${numberedUuid(102)}`, alpha], + ["GET", `/api/folders/${numberedUuid(103)}`, bravo], + [ + "PUT", + `/api/folders/${numberedUuid(102)}`, + first, + 200, + { title: "Renamed Alpha Lantern" }, + ], + ["GET", `/api/folders/${numberedUuid(102)}`, first], + [ + "POST", + `/api/folders/${numberedUuid(102)}/address`, + renamed, + 200, + { url: renamed.url ?? null }, + ], + ["GET", `/api/folders/${numberedUuid(102)}`, renamed], + [ + "PUT", + `/api/folders/${numberedUuid(103)}`, + second, + 200, + { title: "Second Bravo Lantern" }, + ], + ["GET", `/api/folders/${numberedUuid(103)}`, second], + ]; +}; + +const RENAMED_FIRST_TAG = { + ...FIRST_TAG, + name: "Fragile glass", + updatedAt: goClock(3, "512345679"), +}; +const RENAMED_SECOND_TAG = { + ...SECOND_TAG, + name: "Outdoor gear", + updatedAt: goClock(5, "612345680"), +}; + +/** Two tag renames whose update clocks Go spells with nine and then eight fraction digits. */ +export const TAG_UPDATE_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/tags", [FIRST_TAG, SECOND_TAG]], + ["GET", `/api/tags/${numberedUuid(1)}`, FIRST_TAG], + ["GET", `/api/tags/${numberedUuid(2)}`, SECOND_TAG], + [ + "PATCH", + `/api/tags/${numberedUuid(1)}`, + RENAMED_FIRST_TAG, + 200, + { name: "Fragile glass" }, + ], + ["GET", `/api/tags/${numberedUuid(1)}`, RENAMED_FIRST_TAG], + [ + "PATCH", + `/api/tags/${numberedUuid(2)}`, + RENAMED_SECOND_TAG, + 200, + { name: "Outdoor gear" }, + ], + ["GET", `/api/tags/${numberedUuid(2)}`, RENAMED_SECOND_TAG], +]; + +/** The same renames with an archive the gate does not model before the second, which leaves that tag unknown. */ +export const UNKNOWN_TAG_UPDATE_ROWS: readonly TimedCaptureRow[] = [ + ...TAG_UPDATE_ROWS.slice(0, 5), + ["POST", `/api/tags/${numberedUuid(2)}/archive`, { archived: true }, 200, {}], + ...TAG_UPDATE_ROWS.slice(5), +]; + +const synced = (entity: Entity, syncedAt: string): Entity => ({ + ...entity, + syncedAt, +}); + +/** The same renames stamping a second clock, which the later call still shows with nine digits. */ +export const TWO_CLOCK_TAG_UPDATE_ROWS: readonly TimedCaptureRow[] = [ + [ + "GET", + "/api/tags", + [ + synced(FIRST_TAG, "2026-10-01T09:00:00.123456793Z"), + synced(SECOND_TAG, "2026-10-01T10:00:00.223456793Z"), + ], + ], + [ + "GET", + `/api/tags/${numberedUuid(1)}`, + synced(FIRST_TAG, "2026-10-01T09:00:00.123456793Z"), + ], + [ + "GET", + `/api/tags/${numberedUuid(2)}`, + synced(SECOND_TAG, "2026-10-01T10:00:00.223456793Z"), + ], + [ + "PATCH", + `/api/tags/${numberedUuid(1)}`, + synced(RENAMED_FIRST_TAG, goClock(3, "512345681")), + 200, + { name: "Fragile glass" }, + ], + [ + "GET", + `/api/tags/${numberedUuid(1)}`, + synced(RENAMED_FIRST_TAG, goClock(3, "512345681")), + ], + [ + "PATCH", + `/api/tags/${numberedUuid(2)}`, + synced(RENAMED_SECOND_TAG, goClock(5, "612345683")), + 200, + { name: "Outdoor gear" }, + ], + [ + "GET", + `/api/tags/${numberedUuid(2)}`, + synced(RENAMED_SECOND_TAG, goClock(5, "612345683")), + ], +]; + +/** A clock `seconds` after the first exchange with the fraction digits as given. */ +const clockAt = (seconds: number, fraction: string): string => + bookClock(seconds).replace("Z", `.${fraction}Z`); + +/** The same renames whose later call trims the update clock and shows the synced clock's trailing zero. */ +export const MIXED_WIDTH_TAG_UPDATE_ROWS: readonly TimedCaptureRow[] = [ + ...TWO_CLOCK_TAG_UPDATE_ROWS.slice(0, 5), + [ + "PATCH", + `/api/tags/${numberedUuid(2)}`, + synced(RENAMED_SECOND_TAG, clockAt(5, "612345680")), + 200, + { name: "Outdoor gear" }, + ], + [ + "GET", + `/api/tags/${numberedUuid(2)}`, + synced(RENAMED_SECOND_TAG, clockAt(5, "612345680")), + ], +]; + +const SPARE_TAG = tag( + 5, + "Spare", + goClock(7, "412345671"), + goClock(7, "412345683"), +); + +/** The same creates and a third whose creation clock shows nine digits without a trailing zero. */ +export const LATER_FIXED_SIBLING_TAG_ROWS: readonly TimedCaptureRow[] = [ + ...FIXED_SIBLING_TAG_ROWS.slice(0, 7), + ["POST", "/api/tags", SPARE_TAG, 201, { name: "Spare" }], + ["GET", `/api/tags/${numberedUuid(5)}`, SPARE_TAG], + [ + "GET", + "/api/tags", + [FIRST_TAG, SECOND_TAG, HEAVY_TAG, FIXED_CREATED_TAG, SPARE_TAG], + ], +]; + +/** A held tag whose creation clock keeps a trailing zero while its update clock shows eight digits. */ +const FIXED_HELD_TAG = tag( + 1, + "Fragile", + "2026-10-01T09:00:00.123456780Z", + "2026-10-01T09:00:00.12345679Z", +); + +/** One tag create among held tags that show the update clock trimmed and the creation clock at a fixed width. */ +export const FIXED_HELD_TAG_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/tags", [FIXED_HELD_TAG, SECOND_TAG]], + ["GET", `/api/tags/${numberedUuid(1)}`, FIXED_HELD_TAG], + ["GET", `/api/tags/${numberedUuid(2)}`, SECOND_TAG], + ["POST", "/api/tags", HEAVY_TAG, 201, { name: "Heavy" }], + ["GET", `/api/tags/${numberedUuid(3)}`, HEAVY_TAG], + ["GET", "/api/tags", [FIXED_HELD_TAG, SECOND_TAG, HEAVY_TAG]], +]; + +const GLASS_TAG = { + ...FIRST_TAG, + name: "Fragile glass", + updatedAt: clockAt(3, "123456789"), +}; +const GEAR_TAG = { + ...SECOND_TAG, + name: "Outdoor gear", + updatedAt: clockAt(5, "123456780"), +}; +const GLASSWARE_TAG = { + ...GLASS_TAG, + name: "Fragile glassware", + updatedAt: clockAt(7, "12345678"), +}; + +/** Three tag renames whose update clocks show nine digits, nine ending in a zero, then eight. */ +export const MIDDLE_FIXED_TAG_UPDATE_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/tags", [FIRST_TAG, SECOND_TAG]], + ["GET", `/api/tags/${numberedUuid(1)}`, FIRST_TAG], + ["GET", `/api/tags/${numberedUuid(2)}`, SECOND_TAG], + [ + "PATCH", + `/api/tags/${numberedUuid(1)}`, + GLASS_TAG, + 200, + { name: "Fragile glass" }, + ], + ["GET", `/api/tags/${numberedUuid(1)}`, GLASS_TAG], + [ + "PATCH", + `/api/tags/${numberedUuid(2)}`, + GEAR_TAG, + 200, + { name: "Outdoor gear" }, + ], + ["GET", `/api/tags/${numberedUuid(2)}`, GEAR_TAG], + [ + "PATCH", + `/api/tags/${numberedUuid(1)}`, + GLASSWARE_TAG, + 200, + { name: "Fragile glassware" }, + ], + ["GET", `/api/tags/${numberedUuid(1)}`, GLASSWARE_TAG], +]; From 10b83a9b623e42c77d44d21a5745ddef897e492d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:59:09 +0800 Subject: [PATCH 15/37] fix(browser): preserve completed source-map observations Return typed fetch outcomes with completed map coverage when acquisition is cancelled or a response resource fails. Compose those facts with the captured inspection as partial bundle Evidence, preserving exclusions, exact abort causes, and cleanup priority without claiming unperformed script analysis. Keep complete response-body SHA-256 and byte counts when map interpretation fails, while preserving compact failure rows and capacity for later valid maps. Reuse the read owner digest for successful text artifacts and preflight their JSON representation before parsing. Historical failed rows remain importable. Exercise controlled HTTP/stream boundaries, CLI SIGINT, and SDK cancellation with later MCP Evidence retrieval. Share the public cancellation fixture and use one Result API across consumers. Real Chromium and Windows CLI cancellation were not verified. --- docs/browser-observation.md | 27 ++ docs/web-source-location.md | 15 + src/application/BrowserEvidence.ts | 25 +- .../BrowserObservationService.test.ts | 41 +++ src/application/BrowserObservationService.ts | 19 ++ src/browser/CdpBrowserProvider.ts | 55 ++-- src/browser/WebSourceMapFetcher.test.ts | 19 +- src/browser/WebSourceMapFetcher.ts | 269 +++++++++++++----- src/domain/analysisErrorBase.ts | 8 +- src/domain/webBundleAnalysis.ts | 39 ++- .../browserSourceMapCancellationCli.test.ts | 112 ++++++++ .../browser/optionalSourceMapRequire.test.ts | 4 +- .../browser/webSourceMapFetcher.test.ts | 228 ++++++++++----- .../browser/webSourceMapHttpStatus.test.ts | 68 +++-- .../webSourceMapNestedCoordinates.test.ts | 6 +- .../webSourceMapResponseCleanup.test.ts | 4 +- .../webSourceMapResponseIdentity.test.ts | 224 +++++++++++++++ .../browserSourceMapCancellationMcp.test.ts | 150 ++++++++++ .../fixtures/browserSourceMapCancellation.ts | 76 +++++ tests/fixtures/fakeCdpBrowser.ts | 2 + tests/fixtures/fakeCdpBrowserTypes.ts | 3 + tests/fixtures/fetchWebSourceMapsOk.ts | 11 + 22 files changed, 1201 insertions(+), 204 deletions(-) create mode 100644 tests/acceptance/browser/browserSourceMapCancellationCli.test.ts create mode 100644 tests/boundary/browser/webSourceMapResponseIdentity.test.ts create mode 100644 tests/boundary/mcp/browserSourceMapCancellationMcp.test.ts create mode 100644 tests/fixtures/browserSourceMapCancellation.ts create mode 100644 tests/fixtures/fetchWebSourceMapsOk.ts diff --git a/docs/browser-observation.md b/docs/browser-observation.md index f6d13305d1..c73caa90e3 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -598,6 +598,22 @@ Network and console coverage starts only after REA attaches and enables the rele Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation uses `source_map_fetch_timeout_ms` (default 30000) and a 64 MiB aggregate streamed-response budget across redirects. The byte budget has no runtime override. A fetch overrun is an item limitation, not a thrown analysis timeout. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. +A fully read response that fails UTF-8, JSON, mapping, or structural validation +retains `artifact: null`, empty original-source, module-edge, and mapping +arrays, and an explicit failure limitation. Invalid content is `invalid`; +decoder or structural capacity refusal is `fetch_failed`. Its optional +`response_identity` records the complete body's SHA-256 digest, byte count, +and `fetch-response-body-bytes` representation without retaining malformed +text or charging that text to the 32 MiB output budget. These are bytes +delivered by Fetch after HTTP content decoding and before UTF-8 decoding, not +HTTP wire bytes. The identity is absent when the body was not fully read, and +older observations may omit it. A response whose text cannot fit the existing +output budget is likewise `fetch_failed` with text omitted and completed-body +identity; JSON parsing and source-map decoding are not attempted. Successful and partial maps keep their +full text artifacts. Their original text, including an initial BOM or XSSI +line, remains distinct from the parsing view that removes that framing. +Collection reads the body once and does not raise its response or output limits. + When fetching is selected, source-map `excluded` facts preserve the capture ledger's reasons and counts for declarations rejected before download. No disallowed URL details are retained or fetched. `requested` and `processed` @@ -605,6 +621,17 @@ count the admitted fetch requests; exclusions are counted separately. Excluded declarations make the collection unavailable, or partial when maps were retained. An empty collection with no declarations remains `not_requested`. +If the caller cancels while later source maps are being fetched, the failure's +partial Evidence retains the captured page and every completed map item. Its +`normalized_result.kind` is `web-bundle-acquisition` and +`analysis_status` is `not_performed`: no script-analysis result is claimed. +`source_maps.requested` remains the admitted request count, +`source_maps.processed` counts completed items, and any unprocessed requests +have an explicit unknown-results limitation. Capture exclusions remain in the +same collection. The cancellation error keeps the caller's exact reason; +an already observed response resource or cleanup failure keeps its own error +category and cleanup owner. + Source-map items retain the script's `declared_url` separately from the observed `effective_url`. Successful maps use the latter as their source-resolution base. Failed fetches retain the last response URL observed within the approved origins; diff --git a/docs/web-source-location.md b/docs/web-source-location.md index 881a7056c7..41b793ddf6 100644 --- a/docs/web-source-location.md +++ b/docs/web-source-location.md @@ -111,6 +111,10 @@ are made. ## Verification Browser bundle collection uses absolute section offsets for nested inline maps. +If caller cancellation stops browser map fetching after some items complete, +the failed bundle request retains those items with the captured page in partial +Evidence and marks bundle analysis `not_performed`. This is distinct from the +local source-map decoder's failure semantics above. An enclosing column offset contributes only when the nested section starts on that enclosing section's first generated line. Parent boundaries still clip preceding browser mappings when a nested section starts with a gap or has no @@ -136,6 +140,17 @@ requests are left unprocessed and reported as unknown through the top-level items. Malformed UTF-8 map responses are reported as invalid rather than repaired with replacement characters. +For browser fetching, an `invalid` map or structural or decoder capacity +failure has `artifact: null`, empty decoded arrays, and an explicit limitation. +When its bounded response body was read in full, optional `response_identity` +records that body's SHA-256 digest and byte count after HTTP content decoding +and before UTF-8 decoding. It is not a digest of partial reads or a claim that +the map was valid; historical items may omit it. Malformed text is omitted so +it does not consume the 32 MiB inline budget needed by later valid maps. A +response whose text cannot fit that budget is refused before JSON parsing and +source-map decoding, and retains the same +completed-body identity. Successful and partial maps keep their full original +text, including BOM and XSSI framing; only their parsing view removes it. Focused tests cover duplicate mappings, indexed offsets/overlap/local indexes, 32-bit VLQ overflow, nullable content, UTF-16/CRLF positions, immutable artifact diff --git a/src/application/BrowserEvidence.ts b/src/application/BrowserEvidence.ts index a704a6b0e0..449d0a20ca 100644 --- a/src/application/BrowserEvidence.ts +++ b/src/application/BrowserEvidence.ts @@ -15,6 +15,7 @@ import type { } from "../domain/browserObservationSchemas.js"; import type { AnalyzeWebBundleInput, + WebBundleAcquisitionPartialObservation, WebBundleAnalysis, } from "../domain/webBundleAnalysis.js"; import type { @@ -50,6 +51,7 @@ type BrowserEvidenceResult = | BrowserTargetList | WebPageInspection | WebBundleAnalysis + | WebBundleAcquisitionPartialObservation | WebObservationSession | WebMcpDiscovery | BrowserCaptureComparison @@ -65,14 +67,21 @@ type BrowserEvidenceOperation = | "capture_web_screenshot" | "compare_web_screenshots"; +const isBundleAcquisition = ( + result: BrowserEvidenceResult, +): result is WebBundleAcquisitionPartialObservation => + "kind" in result && result.kind === "web-bundle-acquisition"; + /** Create Evidence for one request-scoped external browser observation. */ export const createBrowserEvidence = ( operation: BrowserEvidenceOperation, input: BrowserEvidenceInput, result: BrowserEvidenceResult, provider: ProviderIdentity, -): Evidence => - createEvidence(undefined, provider, { +): Evidence => { + const acquisition = isBundleAcquisition(result); + const observed = acquisition ? result.inspection : result; + return createEvidence(undefined, provider, { predicateType: browserPredicate(operation), operation, parameters: browserParameters(input, result), @@ -80,21 +89,25 @@ export const createBrowserEvidence = ( confidence: "observed", authority: "external-service", environment: - "browser" in result + "browser" in observed ? { - id: `${result.browser.product}@${result.browser.revision}`, + id: `${observed.browser.product}@${observed.browser.revision}`, platform: "unknown", architecture: "unknown", isolation: "none", } : null, limitations: [ - ...result.limitations, - ...("browser" in result + ...observed.limitations, + ...(acquisition && result.source_maps.limitation !== undefined + ? [result.source_maps.limitation] + : []), + ...("browser" in observed ? ["Browser host platform and architecture were not observed."] : []), ], }); +}; const browserParameters = ( input: BrowserEvidenceInput, diff --git a/src/application/BrowserObservationService.test.ts b/src/application/BrowserObservationService.test.ts index d06c0c06ca..b87c4f2a4d 100644 --- a/src/application/BrowserObservationService.test.ts +++ b/src/application/BrowserObservationService.test.ts @@ -163,6 +163,47 @@ it("retains captured page Evidence when bundle analysis fails after capture", as } }); +it("projects completed source-map acquisition without claiming bundle analysis", async () => { + const inspection = passiveNetworkCapture(); + const partial = { + kind: "web-bundle-acquisition" as const, + inspection, + source_maps: { + status: "unavailable" as const, + requested: 1, + processed: 0, + items: [], + limitation: "One requested source map was not processed.", + }, + analysis_status: "not_performed" as const, + }; + const reason = { requested: "stop after capture" }; + const failure = new AnalysisCancelledError("analyze_web_bundle", { + cause: reason, + partialObservation: partial, + }); + const browser: BrowserObservationPort = { + ...provider(async () => ok(report)), + analyzeBundle: async () => err(failure), + }; + const selected = analyzeWebBundleInputSchema.parse({ + ...input, + cdp_endpoint: input.cdp_endpoint.recorded, + target_id: "page", + fetch_source_maps: true, + }); + const result = await analyzeWebBundle(browser, selected); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBe(failure); + expect(result.error.cause).toBe(reason); + expect(parseEvidence(result.error.partialObservation)).toMatchObject({ + operation: "analyze_web_bundle", + parameters: expect.objectContaining({ fetch_source_maps: true }), + normalized_result: partial, + }); +}); + it("retains completed bundle facts as bundle Evidence after cancellation", async () => { const { AnalysisCancelledError } = await import("../domain/analysisErrorCore.js"); diff --git a/src/application/BrowserObservationService.ts b/src/application/BrowserObservationService.ts index 1e32b4ead4..5508a9186e 100644 --- a/src/application/BrowserObservationService.ts +++ b/src/application/BrowserObservationService.ts @@ -4,6 +4,7 @@ import type { ListBrowserTargetsInput, } from "../domain/browserObservation.js"; import { + webBundleAcquisitionPartialObservationSchema, webBundleAnalysisSchema, type AnalyzeWebBundleInput, } from "../domain/webBundleAnalysis.js"; @@ -288,6 +289,24 @@ const retainBrowserObservation = ( errorOperation: "inspect_web_page" | "analyze_web_bundle", ): Result => { const partial = error.partialObservation; + if ( + errorOperation === "analyze_web_bundle" && + partial !== undefined && + "kind" in partial && + partial.kind === "web-bundle-acquisition" + ) { + const acquisition = + webBundleAcquisitionPartialObservationSchema.safeParse(partial); + if (acquisition.success) { + error.partialObservation = createBrowserEvidence( + "analyze_web_bundle", + input, + acquisition.data, + provider.identity(), + ); + return err(error); + } + } if ( errorOperation === "analyze_web_bundle" && partial !== undefined && diff --git a/src/browser/CdpBrowserProvider.ts b/src/browser/CdpBrowserProvider.ts index 4a1de07038..b0a51c72c2 100644 --- a/src/browser/CdpBrowserProvider.ts +++ b/src/browser/CdpBrowserProvider.ts @@ -17,6 +17,7 @@ import { sourceMapsWithCaptureExclusions } from "../domain/webSourceMapCoverage. import type { AnalyzeWebBundleInput, WebBundleAnalysis, + WebSourceMaps, } from "../domain/webBundleAnalysis.js"; import { webObservationSessionSchema, @@ -195,22 +196,43 @@ export class CdpBrowserProvider implements BrowserObservationPort { let captured: CapturedPage | undefined; try { captured = await this.#capture(input, options, "analyze_web_bundle"); - const sourceMaps = input.fetch_source_maps - ? sourceMapsWithCaptureExclusions( - await fetchWebSourceMaps( - captured.sourceMapRequests, - { - ...input, - allowed_origins: - input.allowed_origins.length > 0 - ? input.allowed_origins - : [captured.inspection.target.origin], - }, - options.signal, - ), - captured.inspection.completeness.excluded, + let sourceMaps: WebSourceMaps | undefined; + if (input.fetch_source_maps) { + const fetched = await fetchWebSourceMaps( + captured.sourceMapRequests, + { + ...input, + allowed_origins: + input.allowed_origins.length > 0 + ? input.allowed_origins + : [captured.inspection.target.origin], + }, + options.signal, + ); + if (!fetched.ok) { + const partial = fetched.error.partialObservation; + if ( + partial !== undefined && + "kind" in partial && + partial.kind === "web-source-map-fetch" ) - : undefined; + fetched.error.partialObservation = { + kind: "web-bundle-acquisition", + inspection: captured.inspection, + source_maps: sourceMapsWithCaptureExclusions( + partial.source_maps, + captured.inspection.completeness.excluded, + ), + analysis_status: "not_performed", + }; + else fetched.error.retainPartialObservation(captured.inspection); + return err(fetched.error); + } + sourceMaps = sourceMapsWithCaptureExclusions( + fetched.value, + captured.inspection.completeness.excluded, + ); + } return await this.#bundleAnalyzer.analyze( captured.inspection, sourceMaps ?? { @@ -223,8 +245,7 @@ export class CdpBrowserProvider implements BrowserObservationPort { options.signal, ); } catch (cause: unknown) { - // fetchWebSourceMaps rethrows the exact abort reason. Preserve cancellation - // even when a caller supplied an AnalysisError as that reason. + // Capture and analyzer work can still throw the exact caller reason. if (options.signal?.aborted === true && cause === options.signal.reason) { return err( new AnalysisCancelledError("analyze_web_bundle", { diff --git a/src/browser/WebSourceMapFetcher.test.ts b/src/browser/WebSourceMapFetcher.test.ts index 3f4a3848db..6628c6e714 100644 --- a/src/browser/WebSourceMapFetcher.test.ts +++ b/src/browser/WebSourceMapFetcher.test.ts @@ -4,6 +4,7 @@ import { expect, it } from "vitest"; import { fetchWebSourceMaps } from "./WebSourceMapFetcher.js"; import { analyzeWebBundleInputSchema } from "../domain/webBundleAnalysis.js"; +import { fetchWebSourceMapsOk } from "../../tests/fixtures/fetchWebSourceMapsOk.js"; const input = analyzeWebBundleInputSchema.parse({ cdp_endpoint: "http://127.0.0.1:9222", @@ -18,7 +19,7 @@ const request = { }; it("reports an empty selected request set without starting fetch work", async () => { - const result = await fetchWebSourceMaps([], input, undefined, { + const result = await fetchWebSourceMapsOk([], input, undefined, { fetch: async () => { throw new Error("No source-map request should be made"); }, @@ -37,9 +38,9 @@ it("honors cancellation before returning an empty request set", async () => { const reason = new Error("cancelled before fetching"); controller.abort(reason); - await expect(fetchWebSourceMaps([], input, controller.signal)).rejects.toBe( - reason, - ); + const result = await fetchWebSourceMaps([], input, controller.signal); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.cause).toBe(reason); }); it("cancels a response body that arrives after its fetch host aborts synchronously", async () => { @@ -73,7 +74,9 @@ it("cancels a response body that arrives after its fetch host aborts synchronous }, }); - await expect(pending).rejects.toBe(reason); + const result = await pending; + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.cause).toBe(reason); resolveFetch(new Response(body)); await cancelled; expect(cancellationReason).toBe(reason); @@ -106,7 +109,9 @@ it("observes a fetch rejection that arrives after its host aborts synchronously" process.on("unhandledRejection", onUnhandled); try { - await expect(pending).rejects.toBe(reason); + const result = await pending; + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.cause).toBe(reason); rejectFetch(lateFailure); await new Promise((resolve) => setImmediate(resolve)); expect(unhandled).toBeUndefined(); @@ -149,7 +154,7 @@ it.each([ declaredUrl: "https://app.example.test/maps/app.js.map", fetchUrl: "https://app.example.test/maps/app.js.map", }; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [selectedRequest], fetchInput, undefined, diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 70412511d0..e4d472140b 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -1,9 +1,17 @@ import * as t from "@babel/types"; +import { createHash } from "node:crypto"; import { decodeValidatedSourceMapLeaves } from "../javascript/sourceMaps/DecodedSourceMap.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; -import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; -import type { AnalysisErrorOptions } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { + AnalysisError, + type AnalysisErrorOptions, +} from "../domain/analysisErrorBase.js"; +import { err, ok, type Result } from "../domain/result.js"; import { isUrlLikeModuleSpecifier } from "../domain/webBundleAnalyzerAst.js"; import { analyzeParsedJavaScriptReferences } from "../domain/javascript/javascriptSemanticAnalysis.js"; import { traverseJavaScriptAst } from "../domain/javascript/javascriptSemanticTraversal.js"; @@ -20,7 +28,10 @@ import type { WebSourceMaps, } from "../domain/webBundleAnalysis.js"; import { webSourceMapsSchema } from "../domain/webBundleAnalysis.js"; -import { createWebTextArtifact } from "../domain/webContentArtifact.js"; +import { + createWebTextArtifact, + type WebTextArtifact, +} from "../domain/webContentArtifact.js"; import { safeParseJson } from "../domain/safeJson.js"; import { sourceMapJsonText } from "../domain/sourceMapEnvelope.js"; import { resolveJavaScriptSourceMapUrl } from "../domain/javascript/javascriptSourceMapPaths.js"; @@ -68,6 +79,12 @@ type ParsedSourceMapItem = Extract< WebSourceMapItem, { status: "included" | "partial" } >; +type ResponseIdentity = { + readonly sha256: string; + readonly bytes: number; + readonly representation: "fetch-response-body-bytes"; +}; +type FailedSourceMapStatus = "fetch_failed" | "invalid" | "policy_filtered"; interface SourceMapDecodeContext { readonly signal: AbortSignal | undefined; readonly deadlineAt: number; @@ -85,15 +102,22 @@ export const fetchWebSourceMaps = async ( input: AnalyzeWebBundleInput, signal?: AbortSignal, host: SourceMapFetchHost = { fetch: globalThis.fetch }, -): Promise => { - throwIfAborted(signal); +): Promise> => { + if (signalIsAborted(signal)) + return err( + new AnalysisCancelledError("analyze_web_bundle", { + cause: signal?.reason, + }), + ); if (requests.length === 0) - return webSourceMapsSchema.parse({ - status: "not_requested", - requested: 0, - processed: 0, - items: [], - }); + return ok( + webSourceMapsSchema.parse({ + status: "not_requested", + requested: 0, + processed: 0, + items: [], + }), + ); const items: WebSourceMapItem[] = []; const operationController = new AbortController(); const fetchTimeoutMs = @@ -126,6 +150,33 @@ export const fetchWebSourceMaps = async ( }, }; let limitation: string | undefined; + const completedMaps = (): WebSourceMaps => { + const unprocessed = requests.length - items.length; + const stoppedLimitation = + unprocessed === 0 + ? undefined + : `Source-map acquisition stopped with ${String(unprocessed)} requested map${unprocessed === 1 ? "" : "s"} unprocessed; their results are unknown.`; + const retainedLimitation = [limitation, stoppedLimitation] + .filter((value) => value !== undefined) + .join(" "); + return webSourceMapsSchema.parse({ + status: sourceMapsStatus( + items, + unprocessed > 0 || limitation !== undefined, + ), + requested: requests.length, + processed: items.length, + items, + ...(retainedLimitation.length === 0 + ? {} + : { limitation: retainedLimitation }), + }); + }; + const retainCompleted = (error: AnalysisError): AnalysisError => + error.retainPartialObservation({ + kind: "web-source-map-fetch", + source_maps: completedMaps(), + }); try { for (const request of requests) { if (signal?.aborted === true) throw signal.reason; @@ -152,14 +203,8 @@ export const fetchWebSourceMaps = async ( host, budget, ); - if (item.artifact === null) { - const failureBytes = - sourceMapFailureReservation( - request, - item.effective_url ?? null, - item.limitation, - item.status, - ) + 1; + if (item.status !== "included" && item.status !== "partial") { + const failureBytes = jsonByteLength(item) + 1; if (failureBytes > SOURCE_MAP_OUTPUT_BYTES - expandedBytesBeforeItem) { limitation = sourceMapOutputLimitation( requests.length - items.length, @@ -172,7 +217,23 @@ export const fetchWebSourceMaps = async ( items.push(item); } } catch (cause: unknown) { - if (signal?.aborted === true) throw cause; + if (signal?.aborted === true && cause === signal.reason) + return err( + retainCompleted( + new AnalysisCancelledError("analyze_web_bundle", { + cause: signal.reason, + }), + ), + ); + if (cause instanceof AnalysisError) return err(retainCompleted(cause)); + if (signal?.aborted === true) + return err( + retainCompleted( + new AnalysisCancelledError("analyze_web_bundle", { + cause: signal.reason, + }), + ), + ); if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") limitation = sourceMapOutputLimitation(requests.length - items.length); else throw cause; @@ -180,18 +241,23 @@ export const fetchWebSourceMaps = async ( clearTimeout(timeout); signal?.removeEventListener("abort", abortFromCaller); } - if (signal?.aborted === true) throw signal.reason; - return webSourceMapsSchema.parse({ - status: sourceMapsStatus(items, limitation !== undefined), - requested: requests.length, - processed: items.length, - items, - ...(limitation === undefined ? {} : { limitation }), - }); -}; - -const throwIfAborted = (signal?: AbortSignal): void => { - if (signal?.aborted === true) throw signal.reason; + if (signal?.aborted === true) + return err( + retainCompleted( + new AnalysisCancelledError("analyze_web_bundle", { + cause: signal.reason, + }), + ), + ); + return ok( + webSourceMapsSchema.parse({ + status: sourceMapsStatus(items, limitation !== undefined), + requested: requests.length, + processed: items.length, + items, + ...(limitation === undefined ? {} : { limitation }), + }), + ); }; const sourceMapOutputLimitation = (unprocessed: number): string => @@ -229,7 +295,7 @@ class SourceMapSizeLimitError extends Error { } class SourceMapEncodingError extends Error { - constructor() { + constructor(readonly responseIdentity: ResponseIdentity) { super("Source-map response is not valid UTF-8."); this.name = "SourceMapEncodingError"; } @@ -267,6 +333,7 @@ const fetchOne = async ( "Declared source-map URL is outside the approved exact origins.", ); let effectiveUrl: string | null = null; + let completedResponseIdentity: ResponseIdentity | undefined; try { const fetched = await fetchFollowingApprovedRedirects( request.fetchUrl, @@ -326,14 +393,17 @@ const fetchOne = async ( budget.deadlineAt, budget.deadlineLimitation, ); + const body = await readBoundedText( + response, + budget, + host.maxResponseBytes ?? SOURCE_MAP_RESPONSE_BYTES, + signal, + ); + completedResponseIdentity = body.responseIdentity; return normalizeSourceMap( request, - await readBoundedText( - response, - budget, - host.maxResponseBytes ?? SOURCE_MAP_RESPONSE_BYTES, - signal, - ), + body.text, + body.responseIdentity, fetchedUrl, { signal, @@ -357,6 +427,7 @@ const fetchOne = async ( "invalid", cause.message, effectiveUrl, + cause.responseIdentity, ); return emptySourceMapItem( request, @@ -367,6 +438,7 @@ const fetchOne = async ( ? cause.message : "Source-map fetch or validation failed.", effectiveUrl, + completedResponseIdentity, ); } }; @@ -438,8 +510,16 @@ const readBoundedText = async ( budget: { retainedBytes: number }, maxBytes: number, signal: AbortSignal | undefined, -): Promise => { - if (response.body === null) return ""; +): Promise<{ text: string; responseIdentity: ResponseIdentity }> => { + if (response.body === null) + return { + text: "", + responseIdentity: { + sha256: createHash("sha256").digest("hex"), + bytes: 0, + representation: "fetch-response-body-bytes", + }, + }; const reader = response.body.getReader(); const chunks: Uint8Array[] = []; let bytes = 0; @@ -468,12 +548,20 @@ const readBoundedText = async ( joined.set(chunk, offset); offset += chunk.byteLength; } + const responseIdentity: ResponseIdentity = { + sha256: createHash("sha256").update(joined).digest("hex"), + bytes, + representation: "fetch-response-body-bytes", + }; try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - joined, - ); + return { + text: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + joined, + ), + responseIdentity, + }; } catch { - throw new SourceMapEncodingError(); + throw new SourceMapEncodingError(responseIdentity); } }; @@ -559,9 +647,28 @@ const promiseWithAbort = ( const normalizeSourceMap = ( request: WebSourceMapRequest, text: string, + responseIdentity: ResponseIdentity, fetchedUrl: string, context: SourceMapDecodeContext, ): WebSourceMapItem => { + const retainedBytesBeforeMap = context.budget.expandedBytes; + const textBytes = + jsonByteLength(text) + + artifactMetadataBytes("application/source-map+json", text) - + 4; + const successBytes = + retainedBytesBeforeMap - + SOURCE_MAP_FAILURE_ENVELOPE_BYTES + + SOURCE_MAP_SUCCESS_ENVELOPE_BYTES + + textBytes; + if (successBytes > SOURCE_MAP_OUTPUT_BYTES) + return emptySourceMapItem( + request, + "fetch_failed", + "Source-map response text exceeds the 32 MiB source-map output budget; JSON parsing and source-map decoding were not attempted.", + fetchedUrl, + responseIdentity, + ); const parsedJson = safeParseJson(sourceMapJsonText(text)); if (!parsedJson.ok) return emptySourceMapItem( @@ -569,8 +676,9 @@ const normalizeSourceMap = ( "invalid", "Source-map JSON is not a version 3 map.", fetchedUrl, + responseIdentity, ); - const retainedBytesBeforeMap = context.budget.expandedBytes; + context.budget.expandedBytes = successBytes; try { const check = (): void => checkOperation( @@ -584,16 +692,6 @@ const normalizeSourceMap = ( budget: context.budget, check, }); - context.budget.expandedBytes = - retainedBytesBeforeMap - - SOURCE_MAP_FAILURE_ENVELOPE_BYTES + - SOURCE_MAP_SUCCESS_ENVELOPE_BYTES; - consumeExpandedBytes( - context.budget, - jsonByteLength(text) + - artifactMetadataBytes("application/source-map+json", text) - - 4, - ); const sourcesByRaw = new Map(); const resolvedByRoot = prepareResolvedSources( inspected.leaves, @@ -710,7 +808,7 @@ const normalizeSourceMap = ( ); const parsed = { ...sourceMapContext(request, fetchedUrl), - artifact: createWebTextArtifact(text, "application/source-map+json"), + artifact: sourceMapTextArtifact(text, responseIdentity), original_sources: originalSources, original_module_edges: modules.edges, mappings, @@ -724,16 +822,18 @@ const normalizeSourceMap = ( limitation: `Module edges are incomplete: ${modules.incomplete.length} of ${originalSources.filter(({ artifact }) => artifact !== null).length} original sources could not be parsed in full (${modules.incomplete.join(", ")}).`, }; } catch (cause: unknown) { - // Failed maps publish no artifact or mapping rows. Release their output - // reservation while keeping cumulative fetch and decoder-work accounting. + // Failed interpretations retain completed body identity without consuming + // the collection budget needed for later valid maps. context.budget.expandedBytes = retainedBytesBeforeMap; - if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") + if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") { return emptySourceMapItem( request, "fetch_failed", cause.message, fetchedUrl, + responseIdentity, ); + } if ( context.signal?.aborted === true || cause instanceof SourceMapDeadlineError @@ -747,6 +847,7 @@ const normalizeSourceMap = ( ? cause.message : "Source-map JSON mappings could not be decoded safely.", fetchedUrl, + responseIdentity, ); } }; @@ -887,17 +988,42 @@ const sourceMapContext = ( const emptySourceMapItem = ( request: WebSourceMapRequest, - status: Extract["status"], + status: FailedSourceMapStatus, limitation: string, effectiveUrl: string | null = null, -): Extract => ({ - ...sourceMapContext(request, effectiveUrl), - status, - artifact: null, - original_sources: [], - original_module_edges: [], - mappings: [], - limitation, + responseIdentity?: ResponseIdentity, +): WebSourceMapItem => { + const base = { + ...sourceMapContext(request, effectiveUrl), + artifact: null, + original_sources: [], + original_module_edges: [], + mappings: [], + limitation: + responseIdentity === undefined + ? limitation + : `${limitation} Response text was not retained; completed response body identity is available.`, + }; + return status === "policy_filtered" + ? { ...base, status } + : { + ...base, + status, + ...(responseIdentity === undefined + ? {} + : { response_identity: responseIdentity }), + }; +}; + +const sourceMapTextArtifact = ( + text: string, + responseIdentity: ResponseIdentity, +): WebTextArtifact => ({ + sha256: responseIdentity.sha256, + bytes: responseIdentity.bytes, + media_type: "application/source-map+json", + charset: "utf-8", + text, }); const sanitizeSource = (value: string): string => { @@ -1022,10 +1148,7 @@ const sourceMapFailureReservation = ( request: WebSourceMapRequest, effectiveUrl: string | null = null, limitation?: string, - status: Extract< - WebSourceMapItem, - { readonly artifact: null } - >["status"] = "fetch_failed", + status: FailedSourceMapStatus = "fetch_failed", ): number => SOURCE_MAP_FAILURE_ENVELOPE_BYTES + jsonByteLength(request.declaredUrl) + diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index bd94b43365..a4842d0273 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -268,7 +268,11 @@ import type { RecordedCrashPartialObservation, } from "./native/recordedCrash.js"; import type { WebPageInspection } from "./browserObservationSchemas.js"; -import type { WebBundleAnalysis } from "./webBundleAnalysis.js"; +import type { + WebBundleAcquisitionPartialObservation, + WebBundleAnalysis, + WebSourceMapFetchPartialObservation, +} from "./webBundleAnalysis.js"; import type { WebExecutionObservation } from "./webExecution.js"; import type { WebEventListenersObservation } from "./webEventListeners.js"; @@ -298,6 +302,8 @@ export type AnalysisPartialObservation = | BrowserScenarioPartialObservation | WebPageInspection | WebBundleAnalysis + | WebBundleAcquisitionPartialObservation + | WebSourceMapFetchPartialObservation | WebExecutionObservation | WebEventListenersObservation | JavaScriptRuntimeObservation diff --git a/src/domain/webBundleAnalysis.ts b/src/domain/webBundleAnalysis.ts index d59109773f..9a76a38fe5 100644 --- a/src/domain/webBundleAnalysis.ts +++ b/src/domain/webBundleAnalysis.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { browserExclusionSchema } from "./browserCompleteness.js"; +import { webPageInspectionSchema } from "./browserObservationSchemas.js"; import { javaScriptRouteInventorySchema } from "./javascript/javascriptRouteInventory.js"; import { @@ -280,6 +281,11 @@ const parsedSourceMapShape = { original_module_edges: z.array(originalModuleEdgeSchema), mappings: z.array(sourceMapMappingSchema), }; +const responseIdentitySchema = z.object({ + sha256: z.string().regex(/^[a-f0-9]{64}$/u), + bytes: z.int().min(0), + representation: z.literal("fetch-response-body-bytes"), +}); const sourceMapSchema = z.union([ z.object({ @@ -297,7 +303,18 @@ const sourceMapSchema = z.union([ z.object({ ...sourceMapContextShape, ...parsedSourceMapShape, - status: z.enum(["fetch_failed", "invalid", "policy_filtered"]), + status: z.enum(["fetch_failed", "invalid"]), + artifact: z.null(), + /** Older observations omit this; completed bytes need it only when text is absent. */ + response_identity: responseIdentitySchema.exactOptional(), + original_sources: emptyArraySchema, + original_module_edges: emptyArraySchema, + mappings: emptyArraySchema, + limitation: z.string(), + }), + z.object({ + ...sourceMapContextShape, + status: z.literal("policy_filtered"), artifact: z.null(), original_sources: emptyArraySchema, original_module_edges: emptyArraySchema, @@ -373,6 +390,26 @@ export const webSourceMapsSchema = z export type WebSourceMapItem = z.infer; export type WebSourceMaps = z.infer; +/** Completed fetch facts retained when source-map acquisition stops early. */ +export const webSourceMapFetchPartialObservationSchema = z.object({ + kind: z.literal("web-source-map-fetch"), + source_maps: webSourceMapsSchema, +}); +export type WebSourceMapFetchPartialObservation = z.infer< + typeof webSourceMapFetchPartialObservationSchema +>; + +/** Captured page and completed maps before bundle analysis has begun. */ +export const webBundleAcquisitionPartialObservationSchema = z.object({ + kind: z.literal("web-bundle-acquisition"), + inspection: webPageInspectionSchema, + source_maps: webSourceMapsSchema, + analysis_status: z.literal("not_performed"), +}); +export type WebBundleAcquisitionPartialObservation = z.infer< + typeof webBundleAcquisitionPartialObservationSchema +>; + /** Provider-neutral result of JavaScript bundle reverse engineering. */ export const webBundleAnalysisSchema = z.object({ route_inventory: javaScriptRouteInventorySchema.exactOptional(), diff --git a/tests/acceptance/browser/browserSourceMapCancellationCli.test.ts b/tests/acceptance/browser/browserSourceMapCancellationCli.test.ts new file mode 100644 index 0000000000..52005ab749 --- /dev/null +++ b/tests/acceptance/browser/browserSourceMapCancellationCli.test.ts @@ -0,0 +1,112 @@ +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; + +import { expect } from "vitest"; + +import { analysisErrorProjectionSchema } from "../../../src/contracts/errorSchemas.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { webBundleAcquisitionPartialObservationSchema } from "../../../src/domain/webBundleAnalysis.js"; +import { startSourceMapCancellationBrowser } from "../../fixtures/browserSourceMapCancellation.js"; +import { processTest } from "../../support/process/processFixture.js"; + +// The fake CDP endpoint and HTTP responses exercise the compiled public CLI. +// No Chromium process or real external source-map server is involved. +processTest.skipIf(process.platform === "win32")( + "retains a completed source map and capture exclusions when the next fetch is cancelled", + async ({ processes }) => { + const { browser, mapText, completedMapPath, secondRequest, close } = + await startSourceMapCancellationBrowser(); + const child = processes.spawn( + process.execPath, + [ + resolve("scripts/rea.mjs"), + "analyze-web-bundle", + browser.endpoint, + "allowed-page", + "--allowed-origins", + browser.allowedOrigin, + "--observation-ms", + "0", + "--fetch-source-maps", + "--json", + ], + { env: { PATH: process.env.PATH ?? "" } }, + ); + child.stdin.end(); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + const exited = new Promise<{ + readonly code: number | null; + readonly signal: NodeJS.Signals | null; + }>((resolveExit, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolveExit({ code, signal })); + }); + const deadline = setTimeout(() => child.kill("SIGKILL"), 20_000); + try { + const pendingMapPath = await Promise.race([ + secondRequest, + exited.then(({ code, signal }) => { + throw new Error( + `CLI exited before second map request (${String(code)}, ${String(signal)}): ${stderr}`, + ); + }), + ]); + const retainedMapPath = await completedMapPath; + expect([retainedMapPath, pendingMapPath].sort()).toEqual([ + "/first.js.map", + "/second.js.map", + ]); + expect(child.kill("SIGINT")).toBe(true); + expect(await exited, stderr).toEqual({ code: 130, signal: null }); + const failure = analysisErrorProjectionSchema.parse(JSON.parse(stdout)); + expect(failure.code).toBe("cancelled"); + const partial = parseEvidence(failure.details?.partial_observation); + expect(partial.operation).toBe("analyze_web_bundle"); + const acquisition = webBundleAcquisitionPartialObservationSchema.parse( + partial.normalized_result, + ); + expect(acquisition.analysis_status).toBe("not_performed"); + expect( + acquisition.source_maps, + JSON.stringify(acquisition.source_maps), + ).toMatchObject({ + status: "partial", + requested: 2, + processed: 1, + excluded: [ + { section: "source_maps", reason: "disallowed_origin", count: 1 }, + ], + items: [ + { + status: "included", + declared_url: `${browser.allowedOrigin}${retainedMapPath}`, + artifact: { + text: mapText, + sha256: createHash("sha256").update(mapText).digest("hex"), + }, + }, + ], + }); + expect(acquisition.source_maps.items).toHaveLength(1); + expect(acquisition.inspection.completeness.excluded).toContainEqual({ + section: "source_maps", + reason: "disallowed_origin", + count: 1, + }); + expect(JSON.stringify(partial)).not.toContain("outside.example.test"); + expect(browser.activeConnectionCount()).toBe(0); + expect( + browser.httpRequests + .filter(({ url }) => url.endsWith(".map")) + .map(({ url }) => url), + ).toEqual([retainedMapPath, pendingMapPath]); + } finally { + clearTimeout(deadline); + child.kill("SIGTERM"); + await close(); + } + }, +); diff --git a/tests/boundary/browser/optionalSourceMapRequire.test.ts b/tests/boundary/browser/optionalSourceMapRequire.test.ts index e7e861d777..4d4c7dcf11 100644 --- a/tests/boundary/browser/optionalSourceMapRequire.test.ts +++ b/tests/boundary/browser/optionalSourceMapRequire.test.ts @@ -3,7 +3,7 @@ import { createServer } from "node:http"; import { expect, it } from "vitest"; -import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; +import { fetchWebSourceMapsOk } from "../../fixtures/fetchWebSourceMapsOk.js"; import { analyzeWebBundleInputSchema } from "../../../src/domain/webBundleAnalysis.js"; it("retains unshadowed optional require dependencies from HTTP source-map originals", async () => { @@ -31,7 +31,7 @@ it("retains unshadowed optional require dependencies from HTTP source-map origin throw new TypeError("Expected a TCP listener address"); const origin = `http://127.0.0.1:${String(address.port)}`; const url = `${origin}/app.js.map`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ scriptKey: `scr_${"1".repeat(64)}`, declaredUrl: url, fetchUrl: url }], analyzeWebBundleInputSchema.parse({ cdp_endpoint: origin, diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 796713f0f9..33f5b83e39 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -4,6 +4,8 @@ import { createServer } from "node:http"; import { describe, expect, it } from "vitest"; import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; +import { fetchWebSourceMapsOk } from "../../fixtures/fetchWebSourceMapsOk.js"; +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { analyzeWebBundleInputSchema, webSourceMapsSchema, @@ -68,7 +70,7 @@ it.each(["regular", "indexed"])( if (address === null || typeof address === "string") throw new TypeError("Expected a TCP listener address"); const localOrigin = `http://127.0.0.1:${String(address.port)}`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( texts.map((_, index) => ({ ...request, fetchUrl: `${localOrigin}/${index}`, @@ -159,7 +161,7 @@ it("parses ECMA-426 HTTP source-map prefixes while retaining exact response evid if (address === null || typeof address === "string") throw new TypeError("Expected a TCP listener address"); const localOrigin = `http://127.0.0.1:${String(address.port)}`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( texts.map((_, index) => ({ ...request, fetchUrl: `${localOrigin}/${index}`, @@ -249,7 +251,7 @@ describe("source-map redirect URL resolution", () => { throw new TypeError("Expected a TCP listener address"); const localOrigin = `http://127.0.0.1:${String(address.port)}`; const declaredUrl = `${localOrigin}${initialPath}`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ ...request, fetchUrl: declaredUrl, declaredUrl }], input({ allowed_origins: [localOrigin] }), ); @@ -302,7 +304,7 @@ describe("source-map redirect URL resolution", () => { async (responseStatus, body, itemStatus) => { const effectiveUrl = `${origin}${deliveredPath}`; const calls: string[] = []; - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: (url) => { calls.push(String(url)); return Promise.resolve( @@ -321,6 +323,10 @@ describe("source-map redirect URL resolution", () => { declared_url: request.declaredUrl, effective_url: effectiveUrl, artifact: null, + limitation: + responseStatus === 200 + ? "Source-map JSON is not a version 3 map. Response text was not retained; completed response body identity is available." + : "Source-map server returned HTTP 404.", }); }, ); @@ -328,7 +334,7 @@ describe("source-map redirect URL resolution", () => { it("retains the last observed response URL when the next redirect fetch rejects", async () => { const nextUrl = `${origin}${deliveredPath}`; const calls: string[] = []; - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: (url) => { calls.push(String(url)); return calls.length === 1 @@ -351,7 +357,7 @@ describe("source-map redirect URL resolution", () => { }); it("reports an absent effective URL when the first fetch rejects", async () => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.reject(new TypeError("fixture connection failed")), }); expect(result.items[0]).toMatchObject({ @@ -374,7 +380,7 @@ describe("web source-map fetching and validation: fetching maps and following re sourcesContent: ["import './dependency.ts';\nexport const entry = 1;"], mappings: "AAAAA", }); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: (url, init) => { calls.push({ url: String(url), init }); return Promise.resolve( @@ -433,7 +439,7 @@ describe("web source-map fetching and validation: fetching maps and following re it("reauthorizes every redirect and never contacts a disallowed origin", async () => { const calls: string[] = []; - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: (url) => { calls.push(String(url)); return Promise.resolve( @@ -467,7 +473,7 @@ describe("web source-map fetching and validation: fetching maps and following re "filters a declared fetch URL with %s before any request", async (_description, fetchUrl) => { const calls: string[] = []; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ ...request, fetchUrl }], input(), undefined, @@ -489,7 +495,7 @@ describe("web source-map fetching and validation: fetching maps and following re it("follows an approved redirect chain without an arbitrary hop ceiling", async () => { const calls: string[] = []; - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: (url) => { const current = String(url); calls.push(current); @@ -534,7 +540,7 @@ describe("web source-map fetching and validation: enforcing fetch limits and val throw new TypeError("Expected a TCP listener address"); const localOrigin = `http://127.0.0.1:${String(address.port)}`; const url = `${localOrigin}/map?n=0`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ ...request, fetchUrl: url, declaredUrl: url }], input({ allowed_origins: [localOrigin] }), undefined, @@ -589,7 +595,7 @@ describe("web source-map fetching and validation: enforcing fetch limits and val throw new TypeError("Expected a TCP listener address"); const localOrigin = `http://127.0.0.1:${String(address.port)}`; const url = `${localOrigin}/map`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ ...request, fetchUrl: url, declaredUrl: url }], input({ allowed_origins: [localOrigin] }), undefined, @@ -655,12 +661,101 @@ describe("web source-map fetching and validation: enforcing fetch limits and val await pullStarted; controller.abort(abortReason); - await expect(pending).rejects.toBe(abortReason); + const result = await pending; + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.cause).toBe(abortReason); await cancelled; }); +}); +describe("completed source-map cancellation", () => { + it.each([null, { stopped: "caller" }, new AnalysisCancelledError("test")])( + "retains a completed map when a later response is cancelled with reason %s", + async (reason) => { + let markPullStarted: (() => void) | undefined; + const pullStarted = new Promise((resolve) => { + markPullStarted = resolve; + }); + let markCancelled: (() => void) | undefined; + const cancelled = new Promise((resolve) => { + markCancelled = resolve; + }); + const secondBody = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("{")); + }, + pull() { + markPullStarted?.(); + }, + cancel() { + markCancelled?.(); + }, + }); + const firstBody = JSON.stringify({ + version: 3, + sources: ["src/first.ts"], + names: [], + mappings: "AAAA", + }); + const first = { + ...request, + declaredUrl: `${origin}/first.map`, + fetchUrl: `${origin}/first.map`, + }; + const second = { + ...request, + declaredUrl: `${origin}/second.map`, + fetchUrl: `${origin}/second.map`, + }; + let fetches = 0; + const controller = new AbortController(); + const pending = fetchWebSourceMaps( + [first, second], + input(), + controller.signal, + { + fetch: async () => { + fetches += 1; + return fetches === 1 + ? new Response(firstBody, { status: 200 }) + : new Response(secondBody, { status: 200 }); + }, + }, + ); + + await pullStarted; + controller.abort(reason); + + const result = await pending; + expect(result.ok).toBe(false); + if (result.ok) return; + const failure = result.error; + expect(failure).toBeInstanceOf(AnalysisCancelledError); + expect(failure.cause).toBe(reason); + expect(failure.partialObservation).toMatchObject({ + kind: "web-source-map-fetch", + source_maps: { + requested: 2, + processed: 1, + items: [ + { + status: "included", + artifact: { + text: firstBody, + sha256: createHash("sha256").update(firstBody).digest("hex"), + }, + }, + ], + }, + }); + await cancelled; + }, + ); +}); + +describe("source-map response validation", () => { it("reports malformed source-map JSON as invalid", async () => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(new Response("not-json", { status: 200 })), }); expect(result.items[0]?.status).toBe("invalid"); @@ -693,7 +788,7 @@ describe("selected source-map fetch deadline", () => { throw new TypeError("Expected a TCP listener address"); const localOrigin = `http://127.0.0.1:${String(address.port)}`; const url = `${localOrigin}/map?n=0`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ ...request, fetchUrl: url, declaredUrl: url }], input({ source_map_fetch_timeout_ms: 150, @@ -749,7 +844,7 @@ describe("browser map decoded validation", () => { ...request, scriptKey: `scr_${String(requestIndex + 1).padStart(64, "0")}`, })); - const result = await fetchWebSourceMaps(requests, input(), undefined, { + const result = await fetchWebSourceMapsOk(requests, input(), undefined, { fetch: () => Promise.resolve( new Response(JSON.stringify(maps[index++]), { status: 200 }), @@ -773,7 +868,7 @@ describe("browser map decoded validation", () => { describe("source-map response encoding", () => { it("rejects malformed UTF-8 source-map response bytes", async () => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response(Uint8Array.of(0x7b, 0x22, 0xc3, 0x28, 0x22, 0x7d), { @@ -786,7 +881,8 @@ describe("source-map response encoding", () => { expect(result.items[0]).toMatchObject({ status: "invalid", artifact: null, - limitation: "Source-map response is not valid UTF-8.", + limitation: + "Source-map response is not valid UTF-8. Response text was not retained; completed response body identity is available.", }); }); }); @@ -798,7 +894,7 @@ describe("source-map operation deadline observations", () => { request, { ...request, scriptKey: `scr_${"2".repeat(64)}` }, ]; - const result = await fetchWebSourceMaps(requests, input(), undefined, { + const result = await fetchWebSourceMapsOk(requests, input(), undefined, { timeoutMs: 100, fetch: () => { calls += 1; @@ -828,7 +924,7 @@ describe("indexed source-map offsets", () => { { line: 0 }, { line: 0, column: Number.MAX_SAFE_INTEGER + 1 }, ])("retains an invalid-map item for malformed offset %j", async (offset) => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -863,7 +959,7 @@ describe("web source-map sourcesContent validation", () => { ["a scalar sourcesContent value", { sourcesContent: "source" }], ["a non-string, non-null source entry", { sourcesContent: [42] }], ])("reports %s as invalid", async (_description, content) => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -898,7 +994,7 @@ describe("web source-map sourcesContent validation", () => { ])( "keeps every source declaration when sourcesContent is %s than sources", async (_length, sourcesContent) => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -937,7 +1033,7 @@ describe("web source-map sourcesContent validation", () => { ); it("accepts null source-map source contents", async () => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -967,7 +1063,7 @@ describe("web source-map sourcesContent validation", () => { }); it("keeps unnamed source slots and mapping indices", async () => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1022,7 +1118,7 @@ describe("source-map original dependency syntax", () => { 'import("./dynamic.js");', 'require("./common.cjs");', ].join("\n"); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1048,7 +1144,7 @@ describe("source-map original dependency syntax", () => { "require(`./common.cjs`);", "require(`./${name}.cjs`);", ].join("\n"); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1095,7 +1191,7 @@ describe("source-map original syntax boundaries", () => { ] as const)( "retains literal syntax without inventing module calls", async (text, status, expected) => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1123,7 +1219,7 @@ describe("source-map original syntax boundaries", () => { describe("source-map dependency coverage of hard-to-parse sources", () => { const fetchWithSources = (sources: readonly string[]) => - fetchWebSourceMaps([request], input(), undefined, { + fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1188,7 +1284,7 @@ describe("source-map dependency coverage of hard-to-parse sources", () => { }); it("selects parser mode from the original source URL pathname", async () => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1242,16 +1338,14 @@ describe("source-map dependency coverage of hard-to-parse sources", () => { describe("web source-map collection", () => { it("rejects mapping expansion before the trace-mapping decoder allocates it", async () => { - const response = new Response( - JSON.stringify({ - version: 3, - names: [], - sources: [], - mappings: Array.from({ length: 262_145 }, () => "A").join(","), - }), - { status: 200 }, - ); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const mapText = JSON.stringify({ + version: 3, + names: [], + sources: [], + mappings: Array.from({ length: 262_145 }, () => "A").join(","), + }); + const response = new Response(mapText, { status: 200 }); + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(response), }); @@ -1274,7 +1368,7 @@ describe("web source-map collection", () => { request, { ...request, scriptKey: `scr_${"2".repeat(64)}` }, ]; - const result = await fetchWebSourceMaps(requests, input(), undefined, { + const result = await fetchWebSourceMapsOk(requests, input(), undefined, { fetch: () => Promise.resolve(new Response(mapText, { status: 200 })), }); @@ -1298,7 +1392,7 @@ describe("web source-map collection", () => { version: 3, sections: [{ offset: { line: 0, column: 0 }, map }], }; - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(new Response(JSON.stringify(map), { status: 200 })), }); @@ -1320,7 +1414,7 @@ describe("web source-map collection", () => { sourcesContent: ["export const a = 1"], mappings: Array.from({ length: segmentCount }, () => "AAAA").join(","), }; - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1345,7 +1439,7 @@ describe("web source-map collection", () => { scriptKey: `scr_${String(index + 1).padStart(64, "0")}`, fetchUrl: `${origin}/assets/${String(index)}.js.map`, })); - const result = await fetchWebSourceMaps(requests, input(), undefined, { + const result = await fetchWebSourceMapsOk(requests, input(), undefined, { fetch: (url) => { calls.push(String(url)); return Promise.resolve(validMapResponse()); @@ -1370,7 +1464,7 @@ describe("web source-map collection", () => { sourcesContent: [content], mappings: "AAAA", }); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(new Response(map, { status: 200 })), }); @@ -1394,7 +1488,7 @@ describe("source-map output admission", () => { const effectiveUrl = `${origin}/delivered.map?selection=${"\\".repeat(selectionMiB * 1_024 * 1_024)}`; const secondRequest = { ...request, scriptKey: `scr_${"2".repeat(64)}` }; let calls = 0; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [request, secondRequest], input(), undefined, @@ -1455,7 +1549,7 @@ describe("source-map set output admission", () => { sources: ["x"], mappings: "AAAA", }); - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( requests, input({ allowed_origins: [localOrigin] }), undefined, @@ -1517,7 +1611,7 @@ describe("source-map set output admission", () => { })), ]; let calls = 0; - const result = await fetchWebSourceMaps(requests, input(), undefined, { + const result = await fetchWebSourceMapsOk(requests, input(), undefined, { fetch: () => { calls += 1; return Promise.resolve( @@ -1559,7 +1653,7 @@ describe("expanded source-map evidence budget", () => { sources: [source], mappings: "", }); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(new Response(map, { status: 200 })), }); expect(result.items[0]?.status).toBe("included"); @@ -1584,7 +1678,7 @@ describe("expanded source-map evidence budget", () => { mappings: "AAAA", }); const responses = [malformed, malformed, valid]; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( responses.map((_, index) => ({ ...request, scriptKey: `scr_${String(index + 1).padStart(64, "0")}`, @@ -1616,7 +1710,7 @@ describe("expanded source-map evidence budget", () => { ...request, scriptKey: `scr_${String(index + 1).padStart(64, "0")}`, })); - const result = await fetchWebSourceMaps(requests, input(), undefined, { + const result = await fetchWebSourceMapsOk(requests, input(), undefined, { fetch: () => Promise.resolve(new Response(responses.shift() ?? "", { status: 200 })), }); @@ -1641,7 +1735,7 @@ describe("expanded source-map evidence budget", () => { sources: ["main.ts"], mappings: "", }); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(new Response(map, { status: 200 })), }); @@ -1659,7 +1753,7 @@ describe("expanded source-map evidence budget", () => { sources: ["main.ts"], mappings: "AAAA,AAAA,AAAA", }); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(new Response(map, { status: 200 })), }); @@ -1682,7 +1776,7 @@ describe("expanded source-map evidence budget", () => { sources: Array.from({ length: 65 }, (_, index) => `source-${index}.ts`), mappings: "", }); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(new Response(map, { status: 200 })), }); @@ -1699,18 +1793,16 @@ describe("expanded source-map evidence budget", () => { describe("raw source-map structure preflight", () => { it("budgets source inventory before schema parsing its entries", async () => { - const response = new Response( - JSON.stringify({ - version: 3, - names: [], - sources: Array.from({ length: 262_145 }, (_, index) => - index === 0 ? 42 : "source.js", - ), - mappings: "", - }), - { status: 200 }, - ); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const mapText = JSON.stringify({ + version: 3, + names: [], + sources: Array.from({ length: 262_145 }, (_, index) => + index === 0 ? 42 : "source.js", + ), + mappings: "", + }); + const response = new Response(mapText, { status: 200 }); + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve(response), }); @@ -1745,7 +1837,7 @@ const input = (overrides: Record = {}) => { }; it("keeps bare original module specifiers unresolved", async () => { - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: () => Promise.resolve( new Response( @@ -1817,7 +1909,7 @@ describe("source-map original artifact language metadata", () => { throw new TypeError("Expected a TCP listener address"); const localOrigin = `http://127.0.0.1:${String(address.port)}`; const url = `${localOrigin}/main.js.map`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ ...request, fetchUrl: url, declaredUrl: url }], input({ allowed_origins: [localOrigin] }), AbortSignal.timeout(2_000), @@ -1863,7 +1955,7 @@ describe("deep source-map original syntax", () => { sourcesContent: [source], mappings: "AAAA", }); - const result = await fetchWebSourceMaps([request], input(), undefined, { + const result = await fetchWebSourceMapsOk([request], input(), undefined, { fetch: async () => new Response(text), }); expect(result).toMatchObject({ diff --git a/tests/boundary/browser/webSourceMapHttpStatus.test.ts b/tests/boundary/browser/webSourceMapHttpStatus.test.ts index c5deff2cd7..f9819b309a 100644 --- a/tests/boundary/browser/webSourceMapHttpStatus.test.ts +++ b/tests/boundary/browser/webSourceMapHttpStatus.test.ts @@ -4,6 +4,7 @@ import { createServer } from "node:http"; import { expect, it } from "vitest"; import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; +import { fetchWebSourceMapsOk } from "../../fixtures/fetchWebSourceMapsOk.js"; import { analyzeWebBundleInputSchema, webSourceMapsSchema, @@ -55,7 +56,7 @@ it.each([ fetch_source_maps: true, }); const url = `${origin}/${String(status)}`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [ { scriptKey: `scr_${"1".repeat(64)}`, @@ -102,7 +103,7 @@ it("keeps the declared URL and reports malformed redirect Location evidence", as allowed_origins: ["https://app.example.test"], fetch_source_maps: true, }); - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [ { scriptKey: `scr_${"1".repeat(64)}`, @@ -139,7 +140,7 @@ it.each(["invalid JSON", "invalid UTF-8", "HTTP failure", "response limit"])( const origin = "https://app.example.test"; const declaredUrl = `${origin}/declared.map`; const fetchedUrl = `${origin}/maps/current.map?token=source-value#fragment`; - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ scriptKey: "script", declaredUrl, fetchUrl: declaredUrl }], analyzeWebBundleInputSchema.parse({ cdp_endpoint: "http://127.0.0.1:9222", @@ -188,16 +189,26 @@ it.each(["invalid JSON", "invalid UTF-8", "HTTP failure", "response limit"])( ); it.each([ - { cancellationFails: false, reason: null }, - { cancellationFails: true, reason: new Error("fixture cancel failure") }, - { cancellationFails: true, reason: undefined }, + { cancellationFails: false, reason: null, abortDuringCleanup: false }, + { + cancellationFails: true, + reason: new Error("fixture cancel failure"), + abortDuringCleanup: false, + }, + { cancellationFails: true, reason: undefined, abortDuringCleanup: false }, + { + cancellationFails: true, + reason: new Error("cleanup failed before caller abort"), + abortDuringCleanup: true, + }, ])( - "reserves a fetched URL before reading an over-cap response; cancellation failure=$cancellationFails, reason=$reason", - async ({ cancellationFails, reason }) => { + "reserves a fetched URL before reading an over-cap response; cancellation failure=$cancellationFails, caller abort=$abortDuringCleanup", + async ({ cancellationFails, reason, abortDuringCleanup }) => { const origin = "https://app.example.test"; const fetchedUrl = `${origin}/${"u".repeat(WEB_SOURCE_MAP_LIMITS.outputBytes)}`; let cancelled = false; let read = false; + const controller = new AbortController(); const body = new ReadableStream( { pull(controller) { @@ -211,6 +222,8 @@ it.each([ }, cancel() { cancelled = true; + if (abortDuringCleanup) + controller.abort({ requested: "stop during response cleanup" }); if (cancellationFails) throw reason; }, }, @@ -230,26 +243,33 @@ it.each([ allowed_origins: [origin], fetch_source_maps: true, }), - undefined, + controller.signal, { fetch: () => Promise.resolve(new Response(body)) }, ); if (cancellationFails) { - await expect(pending).rejects.toMatchObject({ - _tag: "AnalysisResourceConstraintError", - message: expect.stringContaining("Expanded source-map evidence"), - cleanupIncomplete: true, - cleanupResources: ["source-map-response"], - cause: expect.objectContaining({ - errors: expect.arrayContaining([reason]), - }), + const result = await pending; + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + message: expect.stringContaining("Expanded source-map evidence"), + cleanupIncomplete: true, + cleanupResources: ["source-map-response"], + cause: expect.objectContaining({ + errors: expect.arrayContaining([reason]), + }), + }, }); } else { await expect(pending).resolves.toMatchObject({ - status: "unavailable", - requested: 1, - processed: 0, - items: [], - limitation: expect.stringContaining("1 requested map"), + ok: true, + value: { + status: "unavailable", + requested: 1, + processed: 0, + items: [], + limitation: expect.stringContaining("1 requested map"), + }, }); } expect(cancelled).toBe(true); @@ -288,10 +308,10 @@ it("charges complete escaped redirect errors and cancels every discarded respons ); }, }; - const control = await fetchWebSourceMaps([request], input, undefined, host); + const control = await fetchWebSourceMapsOk([request], input, undefined, host); expect(control.items[0]?.limitation).toContain(JSON.stringify(location)); expect(control.items[0]?.limitation?.length).toBeGreaterThan(512); - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( Array.from({ length: 1_500 }, () => request), input, undefined, diff --git a/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts b/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts index 7408a48354..a3fa4c163a 100644 --- a/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts +++ b/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { expect, it } from "vitest"; -import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; +import { fetchWebSourceMapsOk } from "../../fixtures/fetchWebSourceMapsOk.js"; import { analyzeWebBundleInputSchema } from "../../../src/domain/webBundleAnalysis.js"; import { traceSourceMap } from "../../../src/javascript/sourceMaps/TraceSourceMap.js"; @@ -335,7 +335,7 @@ it.each(cases)( expect(point.matches).toHaveLength(1); } } - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [{ scriptKey: "scr_" + "1".repeat(64), declaredUrl: url, fetchUrl: url }], analyzeWebBundleInputSchema.parse({ cdp_endpoint: "http://127.0.0.1:9222", @@ -415,7 +415,7 @@ it("rejects a source mapping that crosses an indexed section boundary", async () fetchUrl: url, }; const fetchMap = (map: ReturnType) => - fetchWebSourceMaps([request], input, undefined, { + fetchWebSourceMapsOk([request], input, undefined, { fetch: () => Promise.resolve(new Response(JSON.stringify(map), { status: 200 })), }); diff --git a/tests/boundary/browser/webSourceMapResponseCleanup.test.ts b/tests/boundary/browser/webSourceMapResponseCleanup.test.ts index 88fdafbcb4..a089d05ab8 100644 --- a/tests/boundary/browser/webSourceMapResponseCleanup.test.ts +++ b/tests/boundary/browser/webSourceMapResponseCleanup.test.ts @@ -1,6 +1,6 @@ import { createServer } from "node:http"; import { describe, expect, it, onTestFinished } from "vitest"; -import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; +import { fetchWebSourceMapsOk } from "../../fixtures/fetchWebSourceMapsOk.js"; import { analyzeWebBundleInputSchema } from "../../../src/domain/webBundleAnalysis.js"; describe("source-map discarded HTTP response ownership", () => { @@ -55,7 +55,7 @@ describe("source-map discarded HTTP response ownership", () => { target_id: "page-1", fetch_source_maps: true, }); - const result = await fetchWebSourceMaps( + const result = await fetchWebSourceMapsOk( [ { scriptKey: `scr_${"1".repeat(64)}`, diff --git a/tests/boundary/browser/webSourceMapResponseIdentity.test.ts b/tests/boundary/browser/webSourceMapResponseIdentity.test.ts new file mode 100644 index 0000000000..68fc32fe44 --- /dev/null +++ b/tests/boundary/browser/webSourceMapResponseIdentity.test.ts @@ -0,0 +1,224 @@ +import { createHash } from "node:crypto"; +import { expect, it, onTestFinished, vi } from "vitest"; + +import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; +import { + analyzeWebBundleInputSchema, + webSourceMapsSchema, +} from "../../../src/domain/webBundleAnalysis.js"; + +const origin = "https://maps.example.test"; +const request = (index: number) => ({ + scriptKey: `scr_${String(index).padStart(64, "0")}`, + declaredUrl: `${origin}/${String(index)}.js.map`, + fetchUrl: `${origin}/${String(index)}.js.map`, +}); +const input = analyzeWebBundleInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: [origin], + target_id: "page-1", + fetch_source_maps: true, +}); +const digest = (bytes: Uint8Array | string): string => + createHash("sha256").update(bytes).digest("hex"); + +it.each([ + { label: "an empty body", body: "" }, + { label: "JSON", body: "not-json" }, + { label: "BOM and XSSI framing", body: "\uFEFF)]}'\nnot-json" }, + { + label: "mapping indices", + body: JSON.stringify({ + version: 3, + sources: ["a.js"], + names: [], + mappings: "ACAA", + }), + }, +])( + "retains completed body identity when $label is invalid", + async ({ body }) => { + const result = await fetchWebSourceMaps([request(1)], input, undefined, { + fetch: () => Promise.resolve(new Response(body, { status: 200 })), + }); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + status: "unavailable", + requested: 1, + processed: 1, + items: [ + { + status: "invalid", + artifact: null, + response_identity: { + sha256: digest(body), + bytes: Buffer.byteLength(body), + representation: "fetch-response-body-bytes", + }, + original_sources: [], + original_module_edges: [], + mappings: [], + limitation: expect.stringContaining("Response text was not retained"), + }, + ], + }); + expect(webSourceMapsSchema.parse(result.value)).toEqual(result.value); + }, +); + +it("retains only completed response bytes when UTF-8 decoding fails", async () => { + const body = Uint8Array.of(0x7b, 0x22, 0xc3, 0x28, 0x22, 0x7d); + const result = await fetchWebSourceMaps([request(1)], input, undefined, { + fetch: () => Promise.resolve(new Response(body, { status: 200 })), + }); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + status: "unavailable", + items: [ + { + status: "invalid", + artifact: null, + response_identity: { + sha256: digest(body), + bytes: body.byteLength, + representation: "fetch-response-body-bytes", + }, + original_sources: [], + original_module_edges: [], + mappings: [], + limitation: expect.stringContaining("Response text was not retained"), + }, + ], + }); +}); + +it("retains completed body identity when map interpretation reaches its deadline", async () => { + const body = JSON.stringify({ + version: 3, + sources: ["a.js"], + names: [], + mappings: "AAAA", + }); + const response = new Response(body, { status: 200 }); + const realDecode = TextDecoder.prototype.decode; + let clock = 0; + const clockSpy = vi.spyOn(performance, "now").mockImplementation(() => clock); + const decodeSpy = vi + .spyOn(TextDecoder.prototype, "decode") + .mockImplementation(function ( + this: TextDecoder, + input?: Parameters[0], + options?: Parameters[1], + ) { + const text = realDecode.call(this, input, options); + clock = 1_001; + return text; + }); + try { + const result = await fetchWebSourceMaps([request(1)], input, undefined, { + timeoutMs: 1_000, + fetch: () => Promise.resolve(response), + }); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + status: "unavailable", + requested: 1, + processed: 1, + items: [ + { + status: "fetch_failed", + artifact: null, + response_identity: { + sha256: digest(body), + bytes: Buffer.byteLength(body), + representation: "fetch-response-body-bytes", + }, + original_sources: [], + original_module_edges: [], + mappings: [], + limitation: expect.stringContaining( + "source_map_fetch_timeout_ms deadline", + ), + }, + ], + }); + } finally { + decodeSpy.mockRestore(); + clockSpy.mockRestore(); + } +}); + +it("refuses oversized text before JSON interpretation while retaining body identity and later maps", async () => { + // JSON escaping expands these decoded control characters beyond the fixed + // 32 MiB source-map output allowance while the response stays below 64 MiB. + const oversized = Buffer.alloc(6 * 1024 * 1024, 1); + const oversizedText = oversized.toString("utf8"); + const valid = JSON.stringify({ + version: 3, + sources: ["a.js"], + names: [], + mappings: "AAAA", + }); + const parseSpy = vi.spyOn(JSON, "parse"); + onTestFinished(() => parseSpy.mockRestore()); + let calls = 0; + const result = await fetchWebSourceMaps( + [request(1), request(2)], + input, + undefined, + { + fetch: () => + Promise.resolve( + new Response(calls++ === 0 ? oversized : valid, { status: 200 }), + ), + }, + ); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + status: "partial", + requested: 2, + processed: 2, + items: [ + { + status: "fetch_failed", + artifact: null, + response_identity: { + sha256: digest(oversized), + bytes: oversized.byteLength, + representation: "fetch-response-body-bytes", + }, + mappings: [], + limitation: expect.stringContaining("Response text was not retained"), + }, + { status: "included", artifact: { text: valid }, mappings: [{}] }, + ], + }); + expect(Buffer.byteLength(JSON.stringify(result.value))).toBeLessThanOrEqual( + 32 * 1024 * 1024, + ); + expect(parseSpy.mock.calls.some(([text]) => text === oversizedText)).toBe( + false, + ); + expect(calls).toBe(2); +}); + +it("keeps historical invalid rows with no response identity admissible", () => { + const historical = { + status: "unavailable", + requested: 1, + processed: 1, + items: [ + { + script_key: request(1).scriptKey, + declared_url: request(1).declaredUrl, + status: "invalid", + artifact: null, + original_sources: [], + original_module_edges: [], + mappings: [], + limitation: "Source-map JSON is not a version 3 map.", + }, + ], + }; + expect(webSourceMapsSchema.parse(historical)).toMatchObject(historical); +}); diff --git a/tests/boundary/mcp/browserSourceMapCancellationMcp.test.ts b/tests/boundary/mcp/browserSourceMapCancellationMcp.test.ts new file mode 100644 index 0000000000..813c480b36 --- /dev/null +++ b/tests/boundary/mcp/browserSourceMapCancellationMcp.test.ts @@ -0,0 +1,150 @@ +import { createHash } from "node:crypto"; + +import type { CallToolResult } from "@modelcontextprotocol/server"; +import { expect, it, onTestFinished, vi } from "vitest"; +import { z } from "zod"; + +import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { webBundleAcquisitionPartialObservationSchema } from "../../../src/domain/webBundleAnalysis.js"; +import { ToolResultDelivery } from "../../../src/server/ToolResultDelivery.js"; +import { startSourceMapCancellationBrowser } from "../../fixtures/browserSourceMapCancellation.js"; +import { createDeferred } from "../../fixtures/binarySession.js"; +import { + createApplicationMcpHarness, + type ApplicationMcpHarness, +} from "../../fixtures/applicationMcpHarness.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +it("retains completed source maps through MCP cancellation for later Evidence retrieval", async () => { + const { + browser, + mapText, + completedMapPath, + secondRequest, + releasePendingResponse, + close, + } = await startSourceMapCancellationBrowser(); + const controller = new AbortController(); + let harness: ApplicationMcpHarness | undefined; + onTestFinished(async () => { + controller.abort(); + vi.restoreAllMocks(); + releasePendingResponse(); + try { + await harness?.close(); + } finally { + await close(); + } + }); + const connected = await createApplicationMcpHarness({ + serverOptions: { browserObservation: new CdpBrowserProvider() }, + }); + harness = connected; + + const delivered = createDeferred(); + const original = ToolResultDelivery.prototype.toPartialErrorToolResult; + vi.spyOn( + ToolResultDelivery.prototype, + "toPartialErrorToolResult", + ).mockImplementation(function (this: ToolResultDelivery, ...args) { + const result = original.apply(this, args); + delivered.resolve(result); + return result; + }); + const call = connected.client.callTool( + { + name: "analyze_web_bundle", + arguments: { + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + fetch_source_maps: true, + }, + }, + { signal: controller.signal }, + ); + const pendingMapPath = await Promise.race([ + secondRequest, + call.then( + () => { + throw new Error("MCP completed before the second map request"); + }, + (cause: unknown) => { + throw cause; + }, + ), + ]); + const retainedMapPath = await completedMapPath; + expect([retainedMapPath, pendingMapPath].sort()).toEqual([ + "/first.js.map", + "/second.js.map", + ]); + controller.abort(new Error("Caller cancelled after the first map")); + await expect(call).rejects.toThrow(); + + // SDK cancellation suppresses the reply. This is the server's recording boundary. + const { error } = parseMcpToolError(await delivered.promise); + expect(error.code).toBe("cancelled"); + const reference = z + .object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }) + .parse(error.details).partial_observation; + const reply = await connected.client.callTool({ + name: "get_evidence_bundle", + arguments: { detail: "complete" }, + }); + expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true); + const bundle = toolContract("get_evidence_bundle").outputSchema.parse( + reply.structuredContent, + ).result; + if ("kind" in bundle) throw new Error("Expected complete Evidence bundle"); + const evidence = parseEvidence( + bundle.records.find( + (record) => record.evidence_id === reference.evidence_id, + ), + ); + expect(evidence.operation).toBe("analyze_web_bundle"); + const acquisition = webBundleAcquisitionPartialObservationSchema.parse( + evidence.normalized_result, + ); + expect(acquisition.analysis_status).toBe("not_performed"); + expect(acquisition.source_maps).toMatchObject({ + status: "partial", + requested: 2, + processed: 1, + excluded: [ + { section: "source_maps", reason: "disallowed_origin", count: 1 }, + ], + items: [ + { + status: "included", + declared_url: `${browser.allowedOrigin}${retainedMapPath}`, + artifact: { + text: mapText, + sha256: createHash("sha256").update(mapText).digest("hex"), + }, + }, + ], + }); + expect(acquisition.source_maps.items).toHaveLength(1); + expect(acquisition.inspection.completeness.excluded).toContainEqual({ + section: "source_maps", + reason: "disallowed_origin", + count: 1, + }); + expect(JSON.stringify(evidence)).not.toContain("outside.example.test"); + await expect(connected.client.ping()).resolves.toEqual({}); + expect(browser.activeConnectionCount()).toBe(0); + expect( + browser.httpRequests + .filter(({ url }) => url.endsWith(".map")) + .map(({ url }) => url), + ).toEqual([retainedMapPath, pendingMapPath]); +}); diff --git a/tests/fixtures/browserSourceMapCancellation.ts b/tests/fixtures/browserSourceMapCancellation.ts new file mode 100644 index 0000000000..63a9d7824c --- /dev/null +++ b/tests/fixtures/browserSourceMapCancellation.ts @@ -0,0 +1,76 @@ +import type { ServerResponse } from "node:http"; + +import { createDeferred } from "./binarySession.js"; +import { startFakeCdpBrowser } from "./fakeCdpBrowser.js"; + +/** Two allowed maps and one excluded map, with the second HTTP response held open. */ +export const startSourceMapCancellationBrowser = async () => { + const mapText = JSON.stringify({ + version: 3, + names: [], + sources: ["src/original.js"], + sourcesContent: ["export const value = 1;"], + mappings: "AAAA", + }); + const scriptText = "export const observed = true;"; + const completedMapPath = createDeferred(); + const secondRequest = createDeferred(); + let firstPath: string | undefined; + let heldResponse: ServerResponse | undefined; + const browser = await startFakeCdpBrowser({ + sourceMapReply(url, response) { + if (url !== "/first.js.map" && url !== "/second.js.map") return false; + if (firstPath === undefined) { + firstPath = url; + response.end(mapText); + completedMapPath.resolve(url); + return true; + } + heldResponse = response; + secondRequest.resolve(url); + return true; + }, + commandResult: ({ method }) => + method === "Debugger.getScriptSource" + ? { scriptSource: scriptText } + : undefined, + commandEvents(command, origin) { + if (command.method !== "Debugger.enable") return undefined; + const script = (id: string, sourceMapURL: string) => ({ + method: "Debugger.scriptParsed", + ...(command.sessionId === undefined + ? {} + : { sessionId: command.sessionId }), + params: { + scriptId: id, + url: `${origin}/${id}.js`, + hash: id, + length: Buffer.byteLength(scriptText), + isModule: true, + executionContextId: 1, + scriptLanguage: "JavaScript", + sourceMapURL, + }, + }); + return [ + script("first", "/first.js.map"), + script("second", "/second.js.map"), + script("excluded", "https://outside.example.test/other.js.map"), + ]; + }, + }); + const releasePendingResponse = (): void => { + heldResponse?.destroy(); + }; + return { + browser, + mapText, + completedMapPath: completedMapPath.promise, + secondRequest: secondRequest.promise, + releasePendingResponse, + async close() { + releasePendingResponse(); + await browser.close(); + }, + }; +}; diff --git a/tests/fixtures/fakeCdpBrowser.ts b/tests/fixtures/fakeCdpBrowser.ts index baa12d7107..b359dbae57 100644 --- a/tests/fixtures/fakeCdpBrowser.ts +++ b/tests/fixtures/fakeCdpBrowser.ts @@ -268,6 +268,8 @@ const serveSourceMap = ( response: ServerResponse, options: FakeOptions, ): boolean => { + if (options.sourceMapReply?.(request.url ?? "", response) === true) + return true; if ( options.hangSourceMap === true && request.url?.startsWith("/app.js.map") === true diff --git a/tests/fixtures/fakeCdpBrowserTypes.ts b/tests/fixtures/fakeCdpBrowserTypes.ts index 9203106f66..3ece0256db 100644 --- a/tests/fixtures/fakeCdpBrowserTypes.ts +++ b/tests/fixtures/fakeCdpBrowserTypes.ts @@ -1,5 +1,6 @@ import type { JsonValue } from "../../src/domain/jsonValue.js"; import type { CdpEvent } from "../../src/browser/CdpConnection.js"; +import type { ServerResponse } from "node:http"; export interface FakeCdpCommand { readonly id: number; readonly method: string; @@ -90,6 +91,8 @@ export interface FakeOptions { readonly webSocketWithoutFrames?: boolean; readonly invalidBinaryWebSocketEvent?: boolean; readonly sourceMapBody?: string; + /** Test-owned HTTP response for selected map URLs; true means handled. */ + readonly sourceMapReply?: (url: string, response: ServerResponse) => boolean; /** Accept a source-map request and leave the response open. */ readonly hangSourceMap?: boolean; readonly sourceMapRedirect?: string; diff --git a/tests/fixtures/fetchWebSourceMapsOk.ts b/tests/fixtures/fetchWebSourceMapsOk.ts new file mode 100644 index 0000000000..4fa7da127d --- /dev/null +++ b/tests/fixtures/fetchWebSourceMapsOk.ts @@ -0,0 +1,11 @@ +import { fetchWebSourceMaps } from "../../src/browser/WebSourceMapFetcher.js"; +import type { WebSourceMaps } from "../../src/domain/webBundleAnalysis.js"; + +/** Assert the successful producer branch in tests of completed map collections. */ +export const fetchWebSourceMapsOk = async ( + ...args: Parameters +): Promise => { + const result = await fetchWebSourceMaps(...args); + if (!result.ok) throw result.error; + return result.value; +}; From 4b045e8f2fc00ec218b787ca95cadc1c5f378470 Mon Sep 17 00:00:00 2001 From: Ahmed Date: Sun, 25 Oct 2026 02:10:29 +0500 Subject: [PATCH 16/37] feat(cli): accept stdin for structured JSON request documents (#5486) (#7040) * feat(cli): accept stdin for structured JSON request documents (#5486) * fix(cli): preserve cancellation and admission for JSON stdin --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/application/Utf8JsonInput.ts | 1 + src/cli/electronCommands.ts | 1 + src/cli/wasmCommands.ts | 1 + src/cliBrowserScenarioCommands.ts | 6 +- src/cliJsonInput.ts | 156 ++++++++++++- tests/boundary/cli/cliJsonStdinCli.test.ts | 168 ++++++++++++++ .../boundary/filesystem/cliJsonInput.test.ts | 210 ++++++++++++++++++ tests/fixtures/cliJsonStdinProbe.mjs | 6 + 8 files changed, 539 insertions(+), 10 deletions(-) create mode 100644 tests/boundary/cli/cliJsonStdinCli.test.ts create mode 100644 tests/fixtures/cliJsonStdinProbe.mjs diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.ts index 7b0e5c0ef7..3844fd6454 100644 --- a/src/application/Utf8JsonInput.ts +++ b/src/application/Utf8JsonInput.ts @@ -81,6 +81,7 @@ export const parseUtf8Json = ( return { ok: false, error: "JSON input is not valid UTF-8", + userMessage: "JSON input is not valid UTF-8", cause, }; } diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index 4dd452e852..804d2da92e 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -121,6 +121,7 @@ const registerElectronActiveCommand = ( const input = await parseCliJsonInput( args.inputJson, "capture_electron_scenario", + signal, ); if (!input.ok) return input.error; const parsed = electronActiveObservationInputSchema.safeParse( diff --git a/src/cli/wasmCommands.ts b/src/cli/wasmCommands.ts index 862677ffa3..9748f03e63 100644 --- a/src/cli/wasmCommands.ts +++ b/src/cli/wasmCommands.ts @@ -38,6 +38,7 @@ export const registerWasmCommands = ( const input = await parseCliJsonInput( args.input, "compare_wasm_interfaces", + signal, ); if (!input.ok) return input.error; const result = await compareWasmInterfaces(input.value, { signal }); diff --git a/src/cliBrowserScenarioCommands.ts b/src/cliBrowserScenarioCommands.ts index 59aede588a..20e8923203 100644 --- a/src/cliBrowserScenarioCommands.ts +++ b/src/cliBrowserScenarioCommands.ts @@ -41,7 +41,11 @@ export const registerBrowserScenarioCommands = ( run: ({ args }) => withCommandCancellation((signal) => logCliCommand(logger, CLI_COMMANDS.captureBrowserScenario, async () => { - const input = await parseCliJsonInput(args.inputJson, OPERATION); + const input = await parseCliJsonInput( + args.inputJson, + OPERATION, + signal, + ); if (!input.ok) return input.error; const scenario = browserScenarioSchema.safeParse( resolveCliJsonPaths(input.value, [["browser", "executable_path"]]), diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.ts index c7c0a36d72..d3df8608bb 100644 --- a/src/cliJsonInput.ts +++ b/src/cliJsonInput.ts @@ -1,3 +1,7 @@ +import { addAbortListener } from "node:events"; +import type { Readable } from "node:stream"; +import { getHeapStatistics } from "node:v8"; + import { absoluteCallerPath } from "./filesystem/callerPath.js"; import { sealOwnedJsonInput } from "./application/OwnedJsonInput.js"; @@ -5,23 +9,34 @@ import { JsonInputCancellationFailure, JsonInputPhysicalReadFailure, readJsonInputFile, + nativeJsonParseRequiredBytes, } from "./application/JsonInputFile.js"; import { NonRegularFileReadError } from "./filesystem/RegularFile.js"; import { AnalysisAccessDeniedError, AnalysisCancelledError, AnalysisInputError, + AnalysisResourceConstraintError, } from "./domain/analysisErrorCore.js"; import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; +import { err, ok, type Result } from "./domain/result.js"; import type { JsonValue } from "./domain/jsonValue.js"; import { safeParseJson } from "./domain/safeJson.js"; +import { + JSON_INPUT_RESOURCE_REMEDIATION, + parseUtf8Json, +} from "./application/Utf8JsonInput.js"; -/** Parse inline JSON or one local JSON file, honoring optional caller cancellation. */ +/** Parse inline JSON, standard input ('-'), or one local JSON file, honoring optional caller cancellation. */ export const parseCliJsonInput = async ( value: string, operation: string, signal?: AbortSignal, - options?: { readonly snapshot?: boolean }, + options?: { + readonly snapshot?: boolean; + readonly stdin?: Readable; + readonly maxStdinBytes?: number; + }, ): Promise< | { readonly ok: true; readonly value: unknown } | { readonly ok: false; readonly error: JsonValue } @@ -53,13 +68,16 @@ export const parseCliJsonInput = async ( }, }; } - const parsed = await readJsonInputFile( - value, - operation, - signal, - undefined, - options?.snapshot ?? false, - ); + const parsed = + value === "-" + ? await readCliJsonStdin(operation, signal, options) + : await readJsonInputFile( + value, + operation, + signal, + undefined, + options?.snapshot ?? false, + ); if (parsed.ok && signal?.aborted) return cancelled(signal.reason); return parsed.ok ? { ok: true, value: parsed.value } @@ -106,6 +124,126 @@ export const parseCliJsonInput = async ( } }; +const DEFAULT_MAX_STDIN_BYTES = 8 * 1024 * 1024; + +const readCliJsonStdin = async ( + operation: string, + signal?: AbortSignal, + options?: { + readonly snapshot?: boolean; + readonly stdin?: Readable; + readonly maxStdinBytes?: number; + }, +): Promise< + Result +> => { + const stream = options?.stdin ?? process.stdin; + const maxBytes = options?.maxStdinBytes ?? DEFAULT_MAX_STDIN_BYTES; + const chunks: Buffer[] = []; + let totalBytes = 0; + const requireHeadroom = (): void => { + const heap = getHeapStatistics(); + const requiredBytes = nativeJsonParseRequiredBytes( + totalBytes, + options?.snapshot, + ); + if (requiredBytes > heap.total_available_size) + throw new AnalysisResourceConstraintError( + operation, + "memory", + "Insufficient heap headroom to read and parse the complete JSON stdin document", + { + input_path: "-", + input_bytes: totalBytes, + required_heap_bytes: requiredBytes, + available_heap_bytes: heap.total_available_size, + heap_size_limit_bytes: heap.heap_size_limit, + }, + { remediationAction: JSON_INPUT_RESOURCE_REMEDIATION }, + ); + }; + // Destroying the reader wakes an idle next(); polling inside the loop cannot. + const abort = + signal === undefined + ? undefined + : addAbortListener(signal, () => { + stream.destroy(new JsonInputCancellationFailure(signal.reason)); + }); + try { + try { + for await (const chunk of stream) { + signal?.throwIfAborted(); + totalBytes += + typeof chunk === "string" + ? Buffer.byteLength(chunk) + : chunk.byteLength; + if (totalBytes > maxBytes) + return err( + new AnalysisResourceConstraintError( + operation, + "memory", + "JSON input exceeds the maximum allowed stdin size", + { + input_path: "-", + input_bytes: totalBytes, + max_stdin_bytes: maxBytes, + }, + { remediationAction: JSON_INPUT_RESOURCE_REMEDIATION }, + ), + ); + requireHeadroom(); + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + } catch (cause: unknown) { + if ( + cause instanceof JsonInputCancellationFailure || + (signal?.aborted && isJsonInputCancellation(cause, signal)) + ) + throw cause; + if (cause instanceof AnalysisResourceConstraintError) return err(cause); + throw new JsonInputPhysicalReadFailure(cause); + } finally { + abort?.[Symbol.dispose](); + } + signal?.throwIfAborted(); + // Recheck live headroom before concatenation, UTF-8 decoding and JSON.parse. + requireHeadroom(); + const parsed = parseUtf8Json( + Buffer.concat(chunks, totalBytes), + operation, + "-", + ); + if (!parsed.ok) + return err( + new AnalysisInputError( + operation, + { + cause: parsed.cause, + ...(parsed.userMessage === undefined + ? {} + : { userMessage: parsed.userMessage }), + }, + [ + { + path: [], + reason: "invalid_format", + expected: "JSON", + ...(parsed.userMessage === undefined + ? {} + : { message: parsed.userMessage }), + }, + ], + ), + ); + return options?.snapshot + ? await sealOwnedJsonInput(parsed.value, operation, signal) + : ok(parsed.value); + } catch (cause: unknown) { + if (cause instanceof AnalysisResourceConstraintError) return err(cause); + throw cause; + } +}; + const isJsonInputCancellation = ( cause: unknown, signal: AbortSignal, diff --git a/tests/boundary/cli/cliJsonStdinCli.test.ts b/tests/boundary/cli/cliJsonStdinCli.test.ts new file mode 100644 index 0000000000..1b4150d1a8 --- /dev/null +++ b/tests/boundary/cli/cliJsonStdinCli.test.ts @@ -0,0 +1,168 @@ +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { expect } from "vitest"; + +import { nativeJsonParseRequiredBytes } from "../../../src/application/JsonInputFile.js"; +import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../../../src/application/Utf8JsonInput.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { waitForExit } from "../../support/process/processFixture.js"; + +const probe = pathToFileURL( + resolve("tests/fixtures/cliJsonStdinProbe.mjs"), +).href; + +cliTest.for([ + [Buffer.from("\uFEFF{}"), JSON_BYTE_ORDER_MARK_MESSAGE], + [Buffer.from([0xf0, 0x28, 0x8c, 0xbc]), "JSON input is not valid UTF-8"], + [Buffer.from('{"secret-value":'), undefined], +] as const)( + "projects piped JSON failures through the public CLI (%j)", + async ([input, message], { cli }) => { + const result = await cli.run({ + arguments: ["compare-wasm-interfaces", "-", "--json"], + input, + }); + expect(result.exitCode, result.stderr).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + input_path: "-", + input_reason: "invalid-json", + ...(message === undefined ? {} : { details: { issues: [{ message }] } }), + }); + expect(result.stdout).not.toContain("secret-value"); + }, +); + +cliTest.for([false, true])( + "admits piped JSON against real heap headroom (snapshot: %s)", + async (snapshot, { processes }) => { + const child = processes.spawn( + process.execPath, + [ + resolve("scripts/rea.mjs"), + ...(snapshot + ? ["environment-serve", "-", "--evidence", "{}"] + : ["compare-wasm-interfaces", "-"]), + "--json", + ], + { + env: { + ...process.env, + NODE_OPTIONS: "--max-old-space-size=256", + REA_LOG_LEVEL: "silent", + }, + }, + ); + let stdout = "", + stderr = ""; + let writeFailure: unknown; + child.stdout.on("data", (chunk) => { + stdout += String(chunk); + }); + child.stderr.on("data", (chunk) => { + stderr += String(chunk); + }); + // Admission can close the reader before the producer finishes its write. + child.stdin.on("error", (cause) => { + writeFailure = cause; + }); + const closed = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", () => resolve()); + }); + child.stdin.end(`[${"[],".repeat(1_400_000)}[]]`); + expect(await waitForExit(child, 10_000), stderr).toBe(true); + await closed; + if (writeFailure !== undefined) + expect(writeFailure).toMatchObject({ code: "EPIPE" }); + const result = { + exitCode: child.exitCode, + stderr, + json: JSON.parse(stdout) as unknown, + }; + expect(result.exitCode, result.stderr).toBe(1); + expect(result.json).toMatchObject({ + code: "resource_constraint", + input_path: "-", + input_reason: "too-large", + details: { + resource: "memory", + reported_limits: { + input_path: "-", + required_heap_bytes: expect.any(Number), + available_heap_bytes: expect.any(Number), + }, + }, + }); + const limits = ( + result.json as { + details: { + reported_limits: { + input_bytes: number; + required_heap_bytes: number; + available_heap_bytes: number; + }; + }; + } + ).details.reported_limits; + expect(limits.input_bytes).toBeLessThanOrEqual(8 * 1024 * 1024); + expect(limits.required_heap_bytes).toBe( + nativeJsonParseRequiredBytes(limits.input_bytes, snapshot), + ); + expect(limits.required_heap_bytes).toBeGreaterThan( + limits.available_heap_bytes, + ); + }, +); + +cliTest.skipIf(process.platform === "win32").for( + [ + "capture-browser-scenario", + "capture-electron-scenario", + "compare-wasm-interfaces", + ].flatMap((command) => + ( + [ + ["SIGINT", 130], + ["SIGTERM", 143], + ] as const + ).map(([signal, exitCode]) => ({ command, signal, exitCode })), + ), +)( + "$command cancels an idle stdin reader on $signal", + async ({ command, signal, exitCode }, { processes }) => { + const child = processes.spawn( + process.execPath, + [resolve("scripts/rea.mjs"), command, "-", "--json"], + { + env: { + ...process.env, + NODE_OPTIONS: `--import=${probe}`, + REA_LOG_LEVEL: "silent", + }, + }, + ); + let stdout = "", + stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += String(chunk); + }); + child.stderr.on("data", (chunk) => { + stderr += String(chunk); + }); + const closed = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", () => resolve()); + }); + await expect + .poll(() => stderr.includes("REA_TEST_STDIN_READ"), { timeout: 10_000 }) + .toBe(true); + // Keep the pipe open: EOF must not be what releases the pending read. + child.kill(signal); + expect(await waitForExit(child, 5_000), stderr).toBe(true); + await closed; + expect(child.exitCode, stderr).toBe(exitCode); + expect(JSON.parse(stdout)).toMatchObject({ code: "cancelled" }); + }, +); diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.ts index 097811fcd5..443bcca113 100644 --- a/tests/boundary/filesystem/cliJsonInput.test.ts +++ b/tests/boundary/filesystem/cliJsonInput.test.ts @@ -1,6 +1,8 @@ +import { getEventListeners } from "node:events"; import { execFile } from "node:child_process"; import { chmod, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { Readable } from "node:stream"; import { promisify } from "node:util"; import { describe, expect, it, onTestFinished } from "vitest"; @@ -346,3 +348,211 @@ describe("CLI JSON file cancellation", () => { ).toEqual({ ok: true, value: ["index.js", "main.js", "package.json"] }); }); }); + +describe("CLI JSON standard input", () => { + it("reads valid JSON from stdin when input is '-'", async () => { + const stream = Readable.from(['{"seed":123,"label":"piped"}']); + const result = await parseCliJsonInput("-", "test-input", undefined, { + stdin: stream, + }); + expect(result).toEqual({ + ok: true, + value: { seed: 123, label: "piped" }, + }); + }); + + it("rejects empty stdin with invalid-json", async () => { + const stream = Readable.from([]); + const result = await parseCliJsonInput("-", "test-input", undefined, { + stdin: stream, + }); + expect(result).toMatchObject({ + ok: false, + error: { + code: "invalid_request", + input_path: "-", + input_reason: "invalid-json", + details: { + issues: [{ path: [], reason: "invalid_format" }], + }, + }, + }); + }); + + it("rejects malformed JSON from stdin without printing full payload", async () => { + const stream = Readable.from(["{ this is not valid json"]); + const result = await parseCliJsonInput("-", "test-input", undefined, { + stdin: stream, + }); + expect(result).toMatchObject({ + ok: false, + error: { + code: "invalid_request", + input_path: "-", + input_reason: "invalid-json", + details: { + issues: [{ path: [], reason: "invalid_format" }], + }, + }, + }); + expect(JSON.stringify(result)).not.toContain("this is not valid json"); + }); + + it("rejects oversized stdin with too-large", async () => { + const stream = Readable.from(['{"a":"' + "x".repeat(100) + '"}']); + const result = await parseCliJsonInput("-", "test-input", undefined, { + stdin: stream, + maxStdinBytes: 50, + }); + expect(result).toMatchObject({ + ok: false, + error: { + code: "resource_constraint", + input_path: "-", + input_reason: "too-large", + details: { + resource: "memory", + }, + }, + }); + }); + + it("preserves caller cancellation during stdin read", async () => { + const controller = new AbortController(); + controller.abort("aborted"); + const stream = Readable.from(['{"seed":1}']); + const result = await parseCliJsonInput( + "-", + "test-input", + controller.signal, + { + stdin: stream, + }, + ); + expect(result).toMatchObject({ + ok: false, + error: { + code: "cancelled", + }, + }); + }); +}); + +describe("CLI JSON stdin cancellation and encoding", () => { + it.each([false, true])( + "interrupts an idle stdin read (partial input: %s)", + async (partial) => { + const controller = new AbortController(); + let started!: () => void; + const reading = new Promise((resolve) => { + started = resolve; + }); + const stream = new Readable({ + read() { + started(); + }, + }); + onTestFinished(() => { + stream.destroy(); + }); + if (partial) stream.push('{"partial":'); + const pending = parseCliJsonInput("-", "test-input", controller.signal, { + stdin: stream, + }); + await reading; + // Let any initial chunk be consumed before aborting the next idle read. + await new Promise((resolve) => setImmediate(resolve)); + controller.abort("stop idle stdin"); + await expect(pending).resolves.toMatchObject({ + ok: false, + error: { code: "cancelled" }, + }); + expect(stream.destroyed).toBe(true); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }, + 1_000, + ); + + it.each([ + [Buffer.from("\uFEFF{}"), JSON_BYTE_ORDER_MARK_MESSAGE], + [Buffer.from([0xf0, 0x28, 0x8c, 0xbc]), "JSON input is not valid UTF-8"], + [Buffer.from([0xe2, 0x82]), "JSON input is not valid UTF-8"], + ])( + "preserves safe stdin encoding diagnostics (%j)", + async (bytes, message) => { + expect( + await parseCliJsonInput("-", "test-input", undefined, { + stdin: Readable.from([bytes]), + }), + ).toMatchObject({ + ok: false, + error: { + input_path: "-", + input_reason: "invalid-json", + details: { issues: [{ message }] }, + }, + }); + }, + ); + + it("reads UTF-8 split between chunks and accepts the exact byte limit", async () => { + const bytes = Buffer.from('{"label":"中文🦀"}'); + expect( + await parseCliJsonInput("-", "test-input", undefined, { + stdin: Readable.from([...bytes].map((byte) => Buffer.from([byte]))), + maxStdinBytes: bytes.length, + }), + ).toEqual({ ok: true, value: { label: "中文🦀" } }); + }); + + it("projects cancellation during stdin snapshot sealing", async () => { + const controller = new AbortController(); + const stream = Readable.from([ + JSON.stringify(Array.from({ length: 20_000 }, (_, index) => ({ index }))), + ]); + stream.once("end", () => + setImmediate(() => controller.abort("stop sealing")), + ); + await expect( + parseCliJsonInput("-", "test-input", controller.signal, { + stdin: stream, + snapshot: true, + }), + ).resolves.toMatchObject({ ok: false, error: { code: "cancelled" } }); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("seals stdin snapshots and releases the abort listener on success", async () => { + const controller = new AbortController(); + const result = await parseCliJsonInput( + "-", + "test-input", + controller.signal, + { + stdin: Readable.from(['{"nested":{"value":1}}']), + snapshot: true, + }, + ); + expect(result).toEqual({ ok: true, value: { nested: { value: 1 } } }); + if (result.ok) expect(Object.isFrozen(result.value)).toBe(true); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("preserves stdin physical failure and releases the abort listener", async () => { + const controller = new AbortController(); + const stream = new Readable({ + read() { + this.destroy(Object.assign(new Error("read failed"), { code: "EIO" })); + }, + }); + expect( + await parseCliJsonInput("-", "test-input", controller.signal, { + stdin: stream, + }), + ).toMatchObject({ + ok: false, + error: { input_path: "-", input_reason: "read-failed" }, + }); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); +}); diff --git a/tests/fixtures/cliJsonStdinProbe.mjs b/tests/fixtures/cliJsonStdinProbe.mjs new file mode 100644 index 0000000000..cb9a56af56 --- /dev/null +++ b/tests/fixtures/cliJsonStdinProbe.mjs @@ -0,0 +1,6 @@ +// Observe the production stdin reader without supplying data or closing its pipe. +const iterate = process.stdin[Symbol.asyncIterator]; +process.stdin[Symbol.asyncIterator] = function (...args) { + process.stderr.write("REA_TEST_STDIN_READ\n"); + return iterate.apply(this, args); +}; From d24ad4f55d68cd5e4d2373e7befa51e603817120 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:19:22 +0800 Subject: [PATCH 17/37] docs: make pull request draft status optional --- AGENTS.md | 9 +++++---- CONTRIBUTING.md | 6 +++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3052b0222a..b542f3cfa2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,10 +108,11 @@ distinguish source-derived conclusions, proposed checks, and unavailable coverag Address actionable automated review findings and explain disagreements. Leave the template's **Author readiness** declarations unchecked for the human -owner to complete. Never attest to human review on their behalf. Keep the PR in -draft until the human declares the PR initially ready. Later pushes do not -invalidate that declaration or require another human sign-off. Keep the PR body -and affected validation current as work changes. The readiness check validates +owner to complete. Never attest to human review on their behalf. Draft status +is optional; preserve the PR's current draft status unless the user asks to change +it. Later pushes do not invalidate the readiness declaration or require another +human sign-off. Keep the PR body and affected validation current as work changes. +The readiness check validates the filled declaration; CI and automated review results do not establish that human review occurred, maintainer scope approval, or merge permission. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19f87f919b..e3ff9f31f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,7 +89,11 @@ Human ownership and review remain required when CI or an automated reviewer reports success. Readiness requests maintainer review; it does not grant scope approval or permission to merge. -Draft PRs may request early feedback with open questions and incomplete checks. +Draft status is optional. Draft PRs may request early feedback with open questions +and incomplete checks. Agents preserve the current draft status unless asked to +change it; missing readiness declarations or later updates do not require a PR +to become a draft. + Leave the readiness declaration unchecked until the human owner has completed it. The declaration records initial readiness; later pushes do not invalidate it or require another human sign-off. Keep the PR description and affected From 9548953b479df5ff5fd09f1a0b3b3550809fa6de Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:28:15 +0800 Subject: [PATCH 18/37] refactor(clients): retain inspected command ownership Carry the command identity already parsed during registration inspection into maintenance selection. Keep activation, validity and public status output separate. Validation: fresh build/catalog and full static checks; registration status, update maintenance and setup cleanup ownership suites passed (90 tests). --- src/application/ClientRegistrationStatus.ts | 7 +++- src/application/UpdateMaintenance.ts | 10 +++-- .../clientRegistrationStatus.test.ts | 15 ++++++- .../process-global/updateMaintenance.test.ts | 39 ++++++++++++++++++- 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index 4aa7eb8c03..89093a411b 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -24,6 +24,7 @@ import { import { inspectClientRegistrationCommand, type ClientRegistrationCommandFacts, + type ClientRegistrationCommandIdentity, packageRunnerRegistrationCommand, } from "./ClientRegistrationIdentity.js"; import { @@ -94,6 +95,8 @@ interface ClientRegistrationReadOptions { export interface ClientRegistrationInspection { readonly status: ClientRegistrationStatus; readonly activation: "enabled" | "disabled" | "unknown"; + /** Ownership remains unknown until this inspection has parsed the launcher. */ + readonly commandIdentity: ClientRegistrationCommandIdentity; } /** Project diagnostics from the same inspection used to select maintenance. */ @@ -118,8 +121,9 @@ export const readClientRegistrationInspections = async ( const record = ( status: ClientRegistrationStatus, activation: ClientRegistrationInspection["activation"] = "unknown", + commandIdentity: ClientRegistrationCommandIdentity = "unknown", ): void => { - inspections.push({ status, activation }); + inspections.push({ status, activation, commandIdentity }); }; for (const client of supportedClients( home, @@ -330,6 +334,7 @@ export const readClientRegistrationInspections = async ( : undefined, ), activation, + commandFacts.identity, ); } catch (cause: unknown) { if ( diff --git a/src/application/UpdateMaintenance.ts b/src/application/UpdateMaintenance.ts index 9ea7e5e4c1..1369198321 100644 --- a/src/application/UpdateMaintenance.ts +++ b/src/application/UpdateMaintenance.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; -import { classifyClientRegistrationCommand } from "./ClientRegistrationIdentity.js"; import { readClientRegistrationInspections, type ClientRegistrationStatus, @@ -75,7 +74,11 @@ export const existingMaintenanceScope = async ( ); const clients: string[] = []; const unresolved: ClientRegistrationStatus[] = []; - for (const { status: registration, activation } of registrations) { + for (const { + status: registration, + activation, + commandIdentity, + } of registrations) { if (registration.state === "unknown" || registration.state === "invalid") { unresolved.push(registration); continue; @@ -83,8 +86,7 @@ export const existingMaintenanceScope = async ( if ( activation !== "enabled" || (registration.state !== "aligned" && registration.state !== "stale") || - classifyClientRegistrationCommand(registration.command, entryPoint) !== - "owned" + commandIdentity !== "owned" ) continue; clients.push(registration.client); diff --git a/tests/boundary/filesystem/clientRegistrationStatus.test.ts b/tests/boundary/filesystem/clientRegistrationStatus.test.ts index 79c465b572..53dc032ebe 100644 --- a/tests/boundary/filesystem/clientRegistrationStatus.test.ts +++ b/tests/boundary/filesystem/clientRegistrationStatus.test.ts @@ -6,7 +6,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; -import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { + readClientRegistrationInspections, + readClientRegistrationStatuses, +} from "../../../src/application/ClientRegistrationStatus.js"; import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; import { classifyClientRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; import { existingMaintenanceScope } from "../../../src/application/UpdateMaintenance.js"; @@ -180,6 +183,14 @@ describe("client registration launcher identity", () => { state: "unknown", remediation: expect.stringContaining("registration was retained"), }); + expect(status).not.toHaveProperty("commandIdentity"); + expect(await readClientRegistrationInspections(home, "")).toEqual([ + expect.objectContaining({ + status, + activation: "enabled", + commandIdentity: "unknown", + }), + ]); }); it("does not use the REA process cwd to resolve a client's relative launcher path", async () => { @@ -230,7 +241,9 @@ describe("client registration launcher identity", () => { state: "unknown", }); }); +}); +describe("client registration package version identity", () => { it.each([ ["4.1.0", true], ["v4.1.0", false], diff --git a/tests/process-global/updateMaintenance.test.ts b/tests/process-global/updateMaintenance.test.ts index 48f44bc711..bf03c31f5a 100644 --- a/tests/process-global/updateMaintenance.test.ts +++ b/tests/process-global/updateMaintenance.test.ts @@ -6,6 +6,7 @@ import { err, ok } from "../../src/domain/result.js"; import { runUpdate, type UpdateResult } from "../../src/application/Update.js"; import { PRODUCT_IDENTITY } from "../../src/identity.js"; import { supportedClients } from "../../src/application/SupportedClients.js"; +import { readClientRegistrationInspections } from "../../src/application/ClientRegistrationStatus.js"; import { existingMaintenanceScope, planIntegrationMaintenance, @@ -238,7 +239,7 @@ describe("maintenance activation and layered configuration", () => { }); }); -describe("existing REA integration maintenance", () => { +describe("registration ownership in maintenance", () => { it("selects owned registrations while preserving unconfigured, foreign, and disabled choices", async () => { await writeClient("codex", staleCodex); await writeClient("claude_code", '{"mcpServers":{}}'); @@ -256,6 +257,40 @@ describe("existing REA integration maintenance", () => { servers: { rea: { type: "stdio", command: entryPoint, args: ["mcp"] } }, }), ); + expect( + await readClientRegistrationInspections(home, entryPoint, { + environment: process.env, + }), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: expect.objectContaining({ client: "codex", state: "stale" }), + activation: "enabled", + commandIdentity: "owned", + }), + expect.objectContaining({ + status: expect.objectContaining({ client: "cursor", state: "stale" }), + activation: "enabled", + commandIdentity: "foreign", + }), + expect.objectContaining({ + status: expect.objectContaining({ + client: "opencode", + state: "stale", + }), + activation: "disabled", + commandIdentity: "owned", + }), + expect.objectContaining({ + status: expect.objectContaining({ + client: "claude_code", + state: "missing", + }), + activation: "unknown", + commandIdentity: "unknown", + }), + ]), + ); await expect( existingMaintenanceScope(home, entryPoint, process.env), ).resolves.toEqual({ @@ -264,7 +299,9 @@ describe("existing REA integration maintenance", () => { skillDestinations: [], }); }); +}); +describe("existing REA integration maintenance", () => { it("does not install a missing skill or overwrite an unrelated skill", async () => { await expect( existingMaintenanceScope(home, entryPoint, process.env), From 599099f65670e4e4eb478970e2f7a845eaa53ebd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:28:16 +0800 Subject: [PATCH 19/37] refactor(pe): share optional-header structural facts Decode optional-header declarations once and pass detached directory words, locations and SizeOfHeaders to native consumers. Preserve executable machine admission, raw GPU/resource machine reporting, RVA mapping and certificate file-offset semantics. Validation: fresh build/catalog, full static checks, 466 core/provider fixture tests, 122 workflow tests and 14 selected file-backed PE admission tests passed. Public CLI and actual stdio MCP used authored artifacts; installed external providers were not qualified. Added 17 structural cases and two GPU machine-policy controls. Eager directory decoding costs remain unmeasured. --- src/domain/peMachine.ts | 52 ++--- src/domain/peOptionalHeader.test.ts | 181 ++++++++++++++++++ src/domain/peOptionalHeader.ts | 82 ++++++++ src/native/exports/NativeExportParser.ts | 10 +- src/native/metadata/BinaryContainer.ts | 7 +- src/native/metadata/NativeSectionDirectory.ts | 32 ++-- src/native/metadata/PeImage.ts | 4 +- src/native/pe/PeCertificateParser.ts | 29 +-- src/native/pe/PeImportParser.ts | 14 +- src/native/pe/PeLoadConfigParser.ts | 2 +- src/native/pe/PeResourceLayout.ts | 74 ++++--- tests/boundary/gpu/codeObjects.test.ts | 26 +++ 12 files changed, 394 insertions(+), 119 deletions(-) create mode 100644 src/domain/peOptionalHeader.test.ts create mode 100644 src/domain/peOptionalHeader.ts diff --git a/src/domain/peMachine.ts b/src/domain/peMachine.ts index a08432d2f5..85072455dd 100644 --- a/src/domain/peMachine.ts +++ b/src/domain/peMachine.ts @@ -1,4 +1,8 @@ import type { HeaderArchitecture } from "./binaryTargetTypes.js"; +import { + readPeOptionalHeaderStructure, + type PeOptionalHeaderStructure, +} from "./peOptionalHeader.js"; import { err, ok, type Result } from "./result.js"; export type PeArchitecture = Exclude; @@ -74,14 +78,9 @@ export const peSubjectArchitecture = ( }; /** Optional-header facts shared by admission and native metadata readers. */ -export interface PeOptionalHeaderFacts { - readonly bits: 32 | 64; +export interface PeOptionalHeaderFacts extends PeOptionalHeaderStructure { readonly machine: PeMachine | null; readonly managed: boolean; - readonly dataDirectories: readonly { - readonly virtual_address: number; - readonly size: number; - }[]; } export interface PeHeaderFailure { @@ -94,43 +93,24 @@ export const readPeOptionalHeader = ( optionalHeader: Buffer, machineType: number, ): Result => { - if (optionalHeader.length < 2) - return err({ kind: "malformed", message: "truncated PE optional header" }); - const magic = optionalHeader.readUInt16LE(0); - if (magic !== 0x10b && magic !== 0x20b) - return err({ - kind: "unsupported", - message: "Only PE32 and PE32+ image optional headers are supported", - }); - const bits = magic === 0x20b ? 64 : 32; - const directoryOffset = bits === 64 ? 112 : 96; - if (optionalHeader.length < directoryOffset) - return err({ kind: "malformed", message: "truncated PE optional header" }); - const directoryCount = optionalHeader.readUInt32LE(directoryOffset - 4); - if (directoryCount * 8 > optionalHeader.length - directoryOffset) - return err({ - kind: "malformed", - message: "truncated PE data-directory table", - }); - const dataDirectories = Array.from({ length: directoryCount }, (_, index) => { - const offset = directoryOffset + index * 8; - return { - virtual_address: optionalHeader.readUInt32LE(offset), - size: optionalHeader.readUInt32LE(offset + 4), - }; - }); - const cli = dataDirectories[14]; - const managed = - cli !== undefined && (cli.virtual_address !== 0 || cli.size !== 0); + const parsed = readPeOptionalHeaderStructure(optionalHeader); + if (!parsed.ok) + return err({ kind: parsed.error.kind, message: parsed.error.message }); + const facts = parsed.value; + const cli = facts.dataDirectories[14]; + const managed = cli !== undefined && (cli.address !== 0 || cli.size !== 0); const decoded = decodePeMachine(machineType); // An encoded machine without a CLI declaration remains an unknown CPU to // metadata readers; admission separately rejects unsupported identities. const machine = decoded?.nativeOsOverride && !managed ? null : (decoded ?? null); - if (machine !== null && magic !== peOptionalHeaderMagic(machine.architecture)) + if ( + machine !== null && + facts.magic !== peOptionalHeaderMagic(machine.architecture) + ) return err({ kind: "malformed", message: "PE optional-header magic does not match its architecture", }); - return ok({ bits, machine, managed, dataDirectories }); + return ok({ ...facts, machine, managed }); }; diff --git a/src/domain/peOptionalHeader.test.ts b/src/domain/peOptionalHeader.test.ts new file mode 100644 index 0000000000..c019995951 --- /dev/null +++ b/src/domain/peOptionalHeader.test.ts @@ -0,0 +1,181 @@ +import { expect, it } from "vitest"; +import { readPeOptionalHeader } from "./peMachine.js"; +import { readPeOptionalHeaderStructure } from "./peOptionalHeader.js"; + +const formats = [ + { format: "pe32", magic: 0x10b, bits: 32, directoryOffset: 96 }, + { format: "pe32-plus", magic: 0x20b, bits: 64, directoryOffset: 112 }, +] as const; + +const header = ( + format: (typeof formats)[number], + directoryCount: number, + byteLength = format.directoryOffset + directoryCount * 8, +): Buffer => { + const bytes = Buffer.alloc(byteLength); + bytes.writeUInt16LE(format.magic); + bytes.writeUInt32LE(0x400, 60); + bytes.writeUInt32LE(directoryCount, format.directoryOffset - 4); + return bytes; +}; + +it.each(formats)( + "copies $format declarations and raw directory addresses with their locations", + (format) => { + const bytes = header(format, 5); + bytes.writeUInt32LE(0x2000, format.directoryOffset); + bytes.writeUInt32LE(0x180, format.directoryOffset + 4); + const securityOffset = format.directoryOffset + 4 * 8; + bytes.writeUInt32LE(0x600, securityOffset); + bytes.writeUInt32LE(0x80, securityOffset + 4); + const result = readPeOptionalHeaderStructure(bytes); + if (!result.ok) throw new Error(result.error.message); + expect(result.value).toMatchObject({ + ...format, + sizeOfHeaders: 0x400, + directoryCount: 5, + }); + expect(result.value.dataDirectories).toEqual([ + { address: 0x2000, size: 0x180, offset: format.directoryOffset }, + { address: 0, size: 0, offset: format.directoryOffset + 8 }, + { address: 0, size: 0, offset: format.directoryOffset + 16 }, + { address: 0, size: 0, offset: format.directoryOffset + 24 }, + { address: 0x600, size: 0x80, offset: securityOffset }, + ]); + }, +); + +it.each(formats)( + "keeps $format zero directories and SizeOfHeaders as declarations", + (format) => { + const bytes = header(format, 0); + bytes.writeUInt32LE(0xffffffff, 60); + expect(readPeOptionalHeaderStructure(bytes)).toEqual({ + ok: true, + value: { + ...format, + sizeOfHeaders: 0xffffffff, + directoryCount: 0, + dataDirectories: [], + }, + }); + }, +); + +it.each([ + { + name: "missing magic", + bytes: new Uint8Array(), + kind: "malformed", + reason: "truncated-magic", + }, + { + name: "one magic byte", + bytes: new Uint8Array([0x0b]), + kind: "malformed", + reason: "truncated-magic", + }, + { + name: "unsupported magic before absent fixed fields", + bytes: new Uint8Array([0, 0]), + kind: "unsupported", + reason: "unsupported-magic", + }, + ...formats.map((format) => ({ + name: `truncated ${format.format} fixed fields`, + bytes: header(format, 0).subarray(0, format.directoryOffset - 1), + kind: "malformed", + reason: "truncated-fixed-fields", + })), +])("rejects $name with a structural failure tag", ({ bytes, kind, reason }) => { + expect(readPeOptionalHeaderStructure(bytes)).toMatchObject({ + ok: false, + error: { kind, reason, message: expect.any(String) }, + }); +}); + +it.each(formats)("rejects a partial $format directory entry", (format) => { + const bytes = header(format, 1).subarray(0, format.directoryOffset + 7); + expect(readPeOptionalHeaderStructure(bytes)).toEqual({ + ok: false, + error: { + kind: "malformed", + reason: "truncated-directory-table", + message: "truncated PE data-directory table", + }, + }); +}); + +it.each(formats)( + "refuses the maximum declared $format directory count without its bytes", + (format) => { + const bytes = header(format, 0); + bytes.writeUInt32LE(0xffffffff, format.directoryOffset - 4); + expect(readPeOptionalHeaderStructure(bytes)).toMatchObject({ + ok: false, + error: { kind: "malformed", reason: "truncated-directory-table" }, + }); + }, +); + +it.each(formats)( + "reads every $format directory that fits the maximum COFF optional-header extent", + (format) => { + const byteLength = 0xffff; + const directoryCount = Math.floor( + (byteLength - format.directoryOffset) / 8, + ); + const bytes = header(format, directoryCount, byteLength); + const offset = format.directoryOffset + (directoryCount - 1) * 8; + bytes.writeUInt32LE(0xffffffff, offset); + bytes.writeUInt32LE(0xfffffffe, offset + 4); + const result = readPeOptionalHeaderStructure(bytes); + if (!result.ok) throw new Error(result.error.message); + expect(result.value.directoryCount).toBe(directoryCount); + expect(result.value.dataDirectories).toHaveLength(directoryCount); + expect(result.value.dataDirectories.at(-1)).toEqual({ + address: 0xffffffff, + size: 0xfffffffe, + offset, + }); + }, +); + +it("honors a Uint8Array window and detaches facts from later source changes", () => { + const bytes = header(formats[1], 1); + bytes.writeUInt32LE(0x12345678, 112); + bytes.writeUInt32LE(0x90, 116); + const backing = Buffer.alloc(bytes.length + 12, 0xcc); + bytes.copy(backing, 5); + const window = new Uint8Array( + backing.buffer, + backing.byteOffset + 5, + bytes.length, + ); + const result = readPeOptionalHeaderStructure(window); + if (!result.ok) throw new Error(result.error.message); + backing.fill(0); + expect(result.value).toEqual({ + ...formats[1], + sizeOfHeaders: 0x400, + directoryCount: 1, + dataDirectories: [{ address: 0x12345678, size: 0x90, offset: 112 }], + }); +}); + +it("keeps structural refusal ahead of conflicting machine width", () => { + const bytes = header(formats[0], 0); + bytes.writeUInt32LE(1, 92); + expect(readPeOptionalHeader(bytes, 0x8664)).toEqual({ + ok: false, + error: { kind: "malformed", message: "truncated PE data-directory table" }, + }); + bytes.writeUInt32LE(0, 92); + expect(readPeOptionalHeader(bytes, 0x8664)).toEqual({ + ok: false, + error: { + kind: "malformed", + message: "PE optional-header magic does not match its architecture", + }, + }); +}); diff --git a/src/domain/peOptionalHeader.ts b/src/domain/peOptionalHeader.ts new file mode 100644 index 0000000000..9d27350468 --- /dev/null +++ b/src/domain/peOptionalHeader.ts @@ -0,0 +1,82 @@ +import { err, ok, type Result } from "./result.js"; + +/** Raw data-directory words; entry 4 names file bytes, while resource entry 2 names an RVA. */ +export interface PeDataDirectory { + readonly address: number; + readonly size: number; + /** Byte offset of this entry relative to the optional header. */ + readonly offset: number; +} + +/** Structural facts copied from one declared PE optional header. */ +export interface PeOptionalHeaderStructure { + readonly magic: 0x10b | 0x20b; + readonly bits: 32 | 64; + readonly format: "pe32" | "pe32-plus"; + readonly sizeOfHeaders: number; + readonly directoryOffset: number; + readonly directoryCount: number; + readonly dataDirectories: readonly PeDataDirectory[]; +} + +export interface PeOptionalHeaderFailure { + readonly kind: "malformed" | "unsupported"; + readonly reason: + | "truncated-magic" + | "unsupported-magic" + | "truncated-fixed-fields" + | "truncated-directory-table"; + readonly message: string; +} + +/** Decode structural declarations without admitting a machine, file extent, or directory address. */ +export const readPeOptionalHeaderStructure = ( + bytes: Uint8Array, +): Result => { + if (bytes.byteLength < 2) + return err({ + kind: "malformed", + reason: "truncated-magic", + message: "truncated PE optional header", + }); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const magic = view.getUint16(0, true); + if (magic !== 0x10b && magic !== 0x20b) + return err({ + kind: "unsupported", + reason: "unsupported-magic", + message: "Only PE32 and PE32+ image optional headers are supported", + }); + const bits = magic === 0x20b ? 64 : 32; + const directoryOffset = bits === 64 ? 112 : 96; + if (bytes.byteLength < directoryOffset) + return err({ + kind: "malformed", + reason: "truncated-fixed-fields", + message: "truncated PE optional header", + }); + const directoryCount = view.getUint32(directoryOffset - 4, true); + if (directoryCount > Math.floor((bytes.byteLength - directoryOffset) / 8)) + return err({ + kind: "malformed", + reason: "truncated-directory-table", + message: "truncated PE data-directory table", + }); + const dataDirectories = Array.from({ length: directoryCount }, (_, index) => { + const offset = directoryOffset + index * 8; + return { + address: view.getUint32(offset, true), + size: view.getUint32(offset + 4, true), + offset, + }; + }); + return ok({ + magic, + bits, + format: bits === 64 ? "pe32-plus" : "pe32", + sizeOfHeaders: view.getUint32(60, true), + directoryOffset, + directoryCount, + dataDirectories, + }); +}; diff --git a/src/native/exports/NativeExportParser.ts b/src/native/exports/NativeExportParser.ts index dfe49d776b..20fcb18663 100644 --- a/src/native/exports/NativeExportParser.ts +++ b/src/native/exports/NativeExportParser.ts @@ -98,7 +98,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { architecture: image.architecture, bits: image.bits, }; - if (directory === undefined || directory.virtual_address === 0) + if (directory === undefined || directory.address === 0) return { ...result, export_directory: null, @@ -129,7 +129,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { return value.toString("latin1"); }; - const header = map(directory.virtual_address, PE_EXPORT_DIRECTORY_BYTES); + const header = map(directory.address, PE_EXPORT_DIRECTORY_BYTES); const ordinalBase = reader.u32(header + 16); const functionCount = reader.u32(header + 20); const nameCount = reader.u32(header + 24); @@ -180,7 +180,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { } } - const forwarderEnd = directory.virtual_address + directory.size; + const forwarderEnd = directory.address + directory.size; const retain = createRetention(); const exports: PeExport[] = []; for (let index = 0; index < functionCount; index++) { @@ -189,7 +189,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { if (rva === 0) continue; const ordinal = ordinalBase + index; let target: Pick; - if (rva >= directory.virtual_address && rva < forwarderEnd) { + if (rva >= directory.address && rva < forwarderEnd) { try { target = { address: null, forwarder: cString(rva) }; } catch (cause) { @@ -216,7 +216,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { return { ...result, export_directory: { - rva: directory.virtual_address, + rva: directory.address, size: directory.size, dll_name: dllName, ordinal_base: ordinalBase, diff --git a/src/native/metadata/BinaryContainer.ts b/src/native/metadata/BinaryContainer.ts index c7c92e4818..dad27cfde8 100644 --- a/src/native/metadata/BinaryContainer.ts +++ b/src/native/metadata/BinaryContainer.ts @@ -1,3 +1,5 @@ +import type { PeDataDirectory } from "../../domain/peOptionalHeader.js"; + /** Distinguish corrupt recognized binaries from containers outside parser coverage. */ export class BinaryFormatFailure extends Error { constructor( @@ -120,10 +122,7 @@ interface BinaryContainerFacts { */ readonly pe_header_mapping?: FileMapping; /** PE optional-header data directories, indexed as declared. */ - readonly pe_data_directories?: readonly { - readonly virtual_address: number; - readonly size: number; - }[]; + readonly pe_data_directories?: readonly PeDataDirectory[]; /** Mach-O load-command observation, when available. */ readonly macho_chained_fixups?: boolean; /** Section-name lookup was incomplete; absence by name is not established. */ diff --git a/src/native/metadata/NativeSectionDirectory.ts b/src/native/metadata/NativeSectionDirectory.ts index 6bff184571..85c4f9f4cc 100644 --- a/src/native/metadata/NativeSectionDirectory.ts +++ b/src/native/metadata/NativeSectionDirectory.ts @@ -1,4 +1,5 @@ import { readElfTableDeclarations } from "../../domain/elfDeclaredRanges.js"; +import { readPeOptionalHeaderStructure } from "../../domain/peOptionalHeader.js"; import { BinaryFormatFailure, BinaryReader, @@ -286,23 +287,24 @@ export function* readPeSectionDirectory( const coff = new BinaryReader(yield request(fileSize, BigInt(pe), 24n), true); if (coff.u32(0) !== 0x4550) reject("PE signature is invalid"); const optionalSize = coff.u16(20); - const optional = new BinaryReader( + const optional = readPeOptionalHeaderStructure( yield request(fileSize, BigInt(pe + 24), BigInt(optionalSize)), - true, ); - const magic = optional.u16(0); - const bits = magic === 0x10b ? 32 : magic === 0x20b ? 64 : null; - if (bits === null) - throw new BinaryFormatFailure( - "unsupported", - "Only PE32/PE32+ images are supported", - ); - const minimum = bits === 64 ? 112 : 96; - if ( - optionalSize < minimum || - optional.u32(minimum - 4) * 8 > optionalSize - minimum - ) - reject("PE optional header or data directories are truncated"); + if (!optional.ok) { + switch (optional.error.reason) { + case "truncated-magic": + return reject("16-bit field range is truncated or outside the file"); + case "unsupported-magic": + throw new BinaryFormatFailure( + "unsupported", + "Only PE32/PE32+ images are supported", + ); + case "truncated-fixed-fields": + case "truncated-directory-table": + return reject("PE optional header or data directories are truncated"); + } + } + const { bits } = optional.value; const count = coff.u16(6); const start = pe + 24 + optionalSize; const table = yield request(fileSize, BigInt(start), BigInt(count) * 40n); diff --git a/src/native/metadata/PeImage.ts b/src/native/metadata/PeImage.ts index 285da0ec1f..6524b37068 100644 --- a/src/native/metadata/PeImage.ts +++ b/src/native/metadata/PeImage.ts @@ -82,13 +82,13 @@ export const readPeImage = (bytes: Buffer): PeImage => { ); if (!facts.ok) throw new BinaryFormatFailure(facts.error.kind, facts.error.message); - const { bits, dataDirectories } = facts.value; + const { bits, dataDirectories, sizeOfHeaders } = facts.value; const imageBase = reader.word(optional + (bits === 64 ? 24 : 28), bits); const fileAlignment = reader.u32(optional + 36); const headerMapping: FileMapping = { address: imageBase, offset: 0, - size: Math.min(reader.u32(optional + 60), bytes.length), + size: Math.min(sizeOfHeaders, bytes.length), }; const count = reader.u16(pe + 6); const start = reader.table( diff --git a/src/native/pe/PeCertificateParser.ts b/src/native/pe/PeCertificateParser.ts index 4b55e2f61e..9806626bd3 100644 --- a/src/native/pe/PeCertificateParser.ts +++ b/src/native/pe/PeCertificateParser.ts @@ -54,27 +54,13 @@ export const parsePeCertificates = ( artifact: PeCertificates["artifact"], input: InspectPeCertificatesInput, ): PeCertificates => { - const { pe, optional, magic, directoriesAt, directoryCount, headerSize } = + const { machine, format, dataDirectories, headerSize, optionalHeaderOffset } = readPeHeaders(bytes); - const machine = bytes.readUInt16LE(pe + 4); - const format = magic === 0x20b ? "pe32-plus" : "pe32"; - const securityIndex = 4; - if (directoryCount <= securityIndex) { - return { - artifact, - format, - machine, - table: null, - entries: [], - coverage: { status: "complete", examined_entries: 0, certificates: 0 }, - limitations: [...ABSENT_LIMITATIONS], - }; - } - const directoryAt = optional + directoriesAt + securityIndex * 8; - requireRange(bytes, directoryAt, 8); - const tableOffset = bytes.readUInt32LE(directoryAt); - const tableSize = bytes.readUInt32LE(directoryAt + 4); - if (tableOffset === 0 && tableSize === 0) { + const security = dataDirectories[4]; + if ( + security === undefined || + (security.address === 0 && security.size === 0) + ) { return { artifact, format, @@ -85,6 +71,9 @@ export const parsePeCertificates = ( limitations: [...ABSENT_LIMITATIONS], }; } + const tableOffset = security.address; + const tableSize = security.size; + const directoryAt = optionalHeaderOffset + security.offset; if (tableSize === 0 || tableOffset < headerSize) fail("PE certificate table offset overlaps headers or has zero size."); requireRange(bytes, tableOffset, tableSize); diff --git a/src/native/pe/PeImportParser.ts b/src/native/pe/PeImportParser.ts index fa801d135e..83fc3a76a3 100644 --- a/src/native/pe/PeImportParser.ts +++ b/src/native/pe/PeImportParser.ts @@ -389,12 +389,12 @@ export const parsePeImports = async ( const delayDirectory = image.pe_data_directories?.[PE_DELAY_IMPORT_DIRECTORY]; const imports: PeImportDll[] = []; let importDirectoryResult: PeImports["import_directory"] = null; - if (importDirectory !== undefined && importDirectory.virtual_address !== 0) { + if (importDirectory !== undefined && importDirectory.address !== 0) { const spans: { offset: number; bytes: number }[] = []; for (let index = 0; ; index++) { await checkpoint(); const descriptorRva = - importDirectory.virtual_address + index * IMPORT_DESCRIPTOR_BYTES; + importDirectory.address + index * IMPORT_DESCRIPTOR_BYTES; const descriptor = mapDescriptor( descriptorRva, IMPORT_DESCRIPTOR_BYTES, @@ -453,7 +453,7 @@ export const parsePeImports = async ( }); } importDirectoryResult = { - rva: importDirectory.virtual_address, + rva: importDirectory.address, size: importDirectory.size, locations: spans, }; @@ -463,12 +463,12 @@ export const parsePeImports = async ( const delayImports: PeDelayImportDll[] = []; let delayDirectoryResult: PeImports["delay_import_directory"] = null; - if (delayDirectory !== undefined && delayDirectory.virtual_address !== 0) { + if (delayDirectory !== undefined && delayDirectory.address !== 0) { const spans: { offset: number; bytes: number }[] = []; for (let index = 0; ; index++) { await checkpoint(); const descriptorRva = - delayDirectory.virtual_address + index * DELAY_DESCRIPTOR_BYTES; + delayDirectory.address + index * DELAY_DESCRIPTOR_BYTES; const descriptor = mapDescriptor( descriptorRva, DELAY_DESCRIPTOR_BYTES, @@ -534,7 +534,7 @@ export const parsePeImports = async ( }); } delayDirectoryResult = { - rva: delayDirectory.virtual_address, + rva: delayDirectory.address, size: delayDirectory.size, locations: spans, }; @@ -559,7 +559,7 @@ export const parsePeImports = async ( mappingCompatibility: standardMappingCompatibility, truncatedDescriptor: pefileTruncatedDescriptor, bits, - directoryRva: importDirectory?.virtual_address ?? 0, + directoryRva: importDirectory?.address ?? 0, fileBytes: bytes.length, }), coverage: coverage( diff --git a/src/native/pe/PeLoadConfigParser.ts b/src/native/pe/PeLoadConfigParser.ts index 3dd24b1903..18663530e1 100644 --- a/src/native/pe/PeLoadConfigParser.ts +++ b/src/native/pe/PeLoadConfigParser.ts @@ -164,7 +164,7 @@ export const parsePeLoadConfig = async ( const dataDirectory = (index: number): DirectorySpan | null => { const entry = image.pe_data_directories[index]; if (entry === undefined) return null; - const { virtual_address: rva, size } = entry; + const { address: rva, size } = entry; if (rva === 0 && size === 0) return null; if (rva === 0 || size === 0) throw new BinaryFormatFailure( diff --git a/src/native/pe/PeResourceLayout.ts b/src/native/pe/PeResourceLayout.ts index bf91866ebf..b8fd425031 100644 --- a/src/native/pe/PeResourceLayout.ts +++ b/src/native/pe/PeResourceLayout.ts @@ -1,4 +1,18 @@ import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { + readPeOptionalHeaderStructure, + type PeOptionalHeaderFailure, +} from "../../domain/peOptionalHeader.js"; + +const OPTIONAL_HEADER_FAILURES: Readonly< + Record +> = { + "truncated-magic": "Truncated PE optional header.", + "unsupported-magic": "Unsupported PE optional header magic.", + "truncated-fixed-fields": "Truncated PE data directory count.", + "truncated-directory-table": + "PE directory count exceeds its optional header.", +}; export const peFailure = (message: string): never => { throw new ArtifactReaderFailure("format", message); @@ -71,31 +85,31 @@ export const readPeHeaders = (bytes: Buffer, fileSize = bytes.length) => { if (bytes.readUInt32LE(pe) !== 0x4550) peFailure("Expected a PE signature."); const sectionCount = bytes.readUInt16LE(pe + 6); const optionalSize = bytes.readUInt16LE(pe + 20); - const optional = pe + 24; - requirePeRange(bytes, optional, optionalSize); + const optionalHeaderOffset = pe + 24; + requirePeRange(bytes, optionalHeaderOffset, optionalSize); if (optionalSize < 64) peFailure("Truncated PE optional header."); - const magic = bytes.readUInt16LE(optional); - if (magic !== 0x10b && magic !== 0x20b) - peFailure("Unsupported PE optional header magic."); - const countAt = magic === 0x10b ? 92 : 108; - const directoriesAt = countAt + 4; - if (optionalSize < directoriesAt) - peFailure("Truncated PE data directory count."); - const directoryCount = bytes.readUInt32LE(optional + countAt); - if (directoryCount > Math.floor((optionalSize - directoriesAt) / 8)) - peFailure("PE directory count exceeds its optional header."); - const headerSize = bytes.readUInt32LE(optional + 60); - const table = optional + optionalSize; + const parsed = readPeOptionalHeaderStructure( + bytes.subarray(optionalHeaderOffset, optionalHeaderOffset + optionalSize), + ); + if (!parsed.ok) + return peFailure(OPTIONAL_HEADER_FAILURES[parsed.error.reason]); + const { + sizeOfHeaders: headerSize, + format, + directoryOffset, + dataDirectories, + } = parsed.value; + const table = optionalHeaderOffset + optionalSize; requirePeRange(bytes, table, sectionCount * 40); if (headerSize < table + sectionCount * 40 || headerSize > fileSize) peFailure("Invalid PE SizeOfHeaders."); return { - pe, + machine: bytes.readUInt16LE(pe + 4), + format, sectionCount, - optional, - magic, - directoriesAt, - directoryCount, + optionalHeaderOffset, + directoryOffset, + dataDirectories, headerSize, table, }; @@ -107,12 +121,12 @@ export const readPeResourceLayout = ( fileSize = bytes.length, ) => { const { - pe, + machine, + format, sectionCount, - optional, - magic, - directoriesAt, - directoryCount, + optionalHeaderOffset, + directoryOffset, + dataDirectories, headerSize, table, } = readPeHeaders(bytes, fileSize); @@ -194,14 +208,16 @@ export const readPeResourceLayout = ( requirePeExtent(fileSize, offset, size); return offset; }; - const directoryAt = optional + directoriesAt + 16; - const rva = directoryCount > 2 ? bytes.readUInt32LE(directoryAt) : 0; - const size = directoryCount > 2 ? bytes.readUInt32LE(directoryAt + 4) : 0; + const resource = dataDirectories[2]; + const directoryAt = + optionalHeaderOffset + (resource?.offset ?? directoryOffset + 2 * 8); + const rva = resource?.address ?? 0; + const size = resource?.size ?? 0; if ((rva === 0) !== (size === 0)) peFailure("Inconsistent PE resource directory RVA/size."); return { - format: magic === 0x10b ? ("pe32" as const) : ("pe32-plus" as const), - machine: bytes.readUInt16LE(pe + 4), + format, + machine, directoryAt, rva, size, diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index fa51746f17..cf1f8c9071 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -1079,6 +1079,32 @@ it("retains an embedded CCOB with a corrupted signature as unsupported", async ( }), ); }); +it.each([ + ["unknown COFF machine", 0xffff], + ["x86 machine with a PE32+ optional header", 0x14c], +] as const)( + "discovers PE GPU sections with %s", + async (_description, machine) => { + const bytes = peGpuFixture(); + const pe = bytes.readUInt32LE(60); + bytes.writeUInt16LE(machine, pe + 4); + const result = await inspect(bytes); + if (!result.ok) throw result.error; + expect(result.value.format).toBe("pe"); + expect(result.value.artifact.sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + expect(result.value.coverage).toMatchObject({ + discovery: "complete", + payloads: "complete", + }); + expect(result.value.kernels.map(({ name }) => name)).toEqual([ + "add_one", + "add_one", + ]); + expect(result.value.ptx_artifacts[0]!.text).toBe(ptxFixture); + }, +); it("does not dispatch a high-bit-corrupted DOS signature to the PE parser", async () => { const bytes = peGpuFixture(Buffer.from(ptxFixture)); bytes[0] = bytes[0]! | 0x80; From ae54b96c6fff6b96ea3cf4153dcd51f0096d4734 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:28:16 +0800 Subject: [PATCH 20/37] refactor(tests): use fixture-owned temporary cleanup Remove duplicate root lists and teardown deletion for directories already owned by createTestTempDirectory. Retain independent IDA workspaces, runtime closure, environment resets, permission recovery and explicit deletion assertions. Validation: all nine affected suites passed (126 tests); installed Vitest cleanup ordering reviewed. Provider protocols use authored fixtures, not installed IDA or Ghidra. --- src/ida/IdaWorkspaceThinning.test.ts | 2 - .../claudeDesktopClientConfiguration.test.ts | 10 +-- .../filesystem/clientLifecycle.test.ts | 63 +------------------ .../setupUnavailableProvider.test.ts | 12 +--- .../javascriptRuntimeTargetLocations.test.ts | 12 +--- .../boundary/mcp/bundleComparisonMcp.test.ts | 17 +---- .../ghidra/ghidraTargetSnapshot.test.ts | 13 +--- .../providers/ghidra/ghidraTransport.test.ts | 9 +-- tests/composition/idaProvider.test.ts | 1 - 9 files changed, 11 insertions(+), 128 deletions(-) diff --git a/src/ida/IdaWorkspaceThinning.test.ts b/src/ida/IdaWorkspaceThinning.test.ts index 3af1cd5eef..3d3ee70685 100644 --- a/src/ida/IdaWorkspaceThinning.test.ts +++ b/src/ida/IdaWorkspaceThinning.test.ts @@ -26,7 +26,6 @@ const resolveUniversalTarget = async ( hostArchitecture: "arm64" | "x64", ) => { const directory = await createTestTempDirectory("rea-ida-thin-"); - roots.push(directory); const path = join(directory, "universal"); await writeFile(path, bytes); const resolved = await parseBinaryTarget(path, { hostArchitecture }); @@ -191,7 +190,6 @@ describe("IDA workspace input thinning", () => { it("binds a headless session to the thinned input identity", async () => { const { target } = await universalTarget("arm64"); const root = await createTestTempDirectory("rea-ida-thin-root-"); - roots.push(root); const producer = new RecordingIdaMcp(target, "headless"); const client = new IdaSessionClient( { diff --git a/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts b/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts index 7cc4d2e228..1c64d76348 100644 --- a/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts +++ b/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts @@ -4,7 +4,6 @@ import { mkdir, readFile, readdir, - rm, stat, symlink, writeFile, @@ -37,19 +36,12 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const command = npxRegistrationCommand("win32"); const CONFIG = "claude_desktop_config.json"; -const roots: string[] = []; beforeEach(clearClientLocationEnvironment); -afterEach(async () => { - vi.unstubAllEnvs(); - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); -}); +afterEach(() => vi.unstubAllEnvs()); /** A Windows profile with separate roaming and local AppData roots. */ const windowsProfile = async () => { const home = await createTestTempDirectory("rea-claude-desktop-"); - roots.push(home); const env = { APPDATA: join(home, "AppData", "Roaming"), LOCALAPPDATA: join(home, "AppData", "Local"), diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index a5794f2f36..0347150674 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -34,20 +34,13 @@ import { type UninstallFileSystem, } from "../../../src/application/Uninstall.js"; -const roots: string[] = []; const ownedSkillManifest = `---\nname: ${PRODUCT_IDENTITY.skillName}\n---\n# REA\n`; beforeEach(clearClientLocationEnvironment); -afterEach(async () => { - vi.unstubAllEnvs(); - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); -}); +afterEach(() => vi.unstubAllEnvs()); describe("client configuration filesystem lifecycle", () => { it("detects every supported client and skips absent clients", async () => { const home = await createTestTempDirectory("rea-detect-"); - roots.push(home); for (const client of supportedClients(home)) if (client.markerPath !== undefined) await mkdir(client.markerPath, { recursive: true }); @@ -84,13 +77,11 @@ describe("client configuration filesystem lifecycle", () => { ).toBe(join(home, ".claude.json")); expect(detected.find(({ name }) => name === "devin")?.format).toBe("json"); const emptyHome = await createTestTempDirectory("rea-empty-"); - roots.push(emptyHome); expect(await detectClients(emptyHome)).toEqual([]); }); it("preserves unrelated Codex TOML and supports an installed command", async () => { const home = await createTestTempDirectory("rea-toml-"); - roots.push(home); const configPath = join(home, ".codex/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -131,7 +122,6 @@ describe("client configuration filesystem lifecycle", () => { it("does not rewrite an equivalent Codex registration with reordered environment keys", async () => { const home = await createTestTempDirectory("rea-toml-"); - roots.push(home); const configPath = join(home, ".codex/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -161,7 +151,6 @@ describe("client configuration filesystem lifecycle", () => { describe("Codex MCP tool deadlines", () => { it("derives the Codex tool deadline from an explicit Ghidra startup allowance", async () => { const home = await createTestTempDirectory("rea-codex-long-timeout-"); - roots.push(home); const configPath = join(home, ".codex/config.toml"); await mkdir(dirname(configPath), { recursive: true }); expect( @@ -181,7 +170,6 @@ describe("Codex MCP tool deadlines", () => { it("preserves positive caller-selected Codex startup and tool deadlines", async () => { const home = await createTestTempDirectory("rea-codex-custom-timeout-"); - roots.push(home); const configPath = join(home, ".codex/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -204,7 +192,6 @@ describe("Codex MCP tool deadlines", () => { describe("client configuration filesystem lifecycle", () => { it("updates a symlink target without replacing the TOML config symlink", async () => { const home = await createTestTempDirectory("rea-toml-symlink-"); - roots.push(home); const configPath = join(home, ".codex/config.toml"); const targetPath = join(home, "managed-config.toml"); const original = 'model = "gpt-5"\n'; @@ -226,7 +213,6 @@ describe("client configuration filesystem lifecycle", () => { it("fails before mutation when a TOML config symlink is dangling", async () => { const home = await createTestTempDirectory("rea-toml-symlink-"); - roots.push(home); const configPath = join(home, ".codex/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await symlink(join(home, "missing-config.toml"), configPath); @@ -248,7 +234,6 @@ describe("client configuration filesystem lifecycle", () => { describe("Grok Build server table edits", () => { it("edits only the Grok Build REA tables and removes them on uninstall", async () => { const home = await createTestTempDirectory("rea-grok-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = [ @@ -368,7 +353,6 @@ describe("Grok Build server table edits", () => { describe("Grok Build disabled server list", () => { it("removes rea from disabled_mcp_servers and leaves other names", async () => { const home = await createTestTempDirectory("rea-grok-disabled-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -428,7 +412,6 @@ describe("Grok Build disabled server list", () => { it("removes multiline rea entries from disabled_mcp_servers", async () => { const home = await createTestTempDirectory("rea-grok-disabled-multiline-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -474,7 +457,6 @@ describe("Grok Build disabled server list", () => { describe("Grok Build text that resembles a server table", () => { it("preserves Grok text that only looks like an REA table", async () => { const home = await createTestTempDirectory("rea-grok-preserve-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = [ @@ -556,7 +538,6 @@ describe("Grok Build text that resembles a server table", () => { it("replaces spaced and escaped Grok REA headers without duplicating them", async () => { const home = await createTestTempDirectory("rea-grok-alias-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -603,7 +584,6 @@ describe("Grok Build text that resembles a server table", () => { describe("Grok Build multiline and escaped headers", () => { it("preserves a multiline string opened on the line that closed the previous one", async () => { const home = await createTestTempDirectory("rea-grok-multiline-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -659,7 +639,6 @@ describe("Grok Build multiline and escaped headers", () => { it("replaces a hex-escaped Grok REA header and keeps a hex-escaped foreign name", async () => { const home = await createTestTempDirectory("rea-grok-hex-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = [ @@ -712,7 +691,6 @@ describe("Grok Build multiline and escaped headers", () => { describe("Grok Build inline and dotted assignments", () => { it("updates and removes inline and dotted Grok REA assignments", async () => { const home = await createTestTempDirectory("rea-grok-assign-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const inline = [ @@ -818,7 +796,6 @@ describe("Grok Build inline and dotted assignments", () => { describe("Grok Build closed inline tables", () => { it("splices REA into a closed mcp_servers inline table", async () => { const home = await createTestTempDirectory("rea-grok-inline-table-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = [ @@ -898,7 +875,6 @@ describe("Grok Build closed inline tables", () => { it("keeps a quote that belongs to a multiline string closer", async () => { const home = await createTestTempDirectory("rea-grok-quotes-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -938,7 +914,6 @@ describe("Grok Build closed inline tables", () => { describe("Grok Build scalar and nested values", () => { it("preserves a basic string that uses the escape smol-toml accepts", async () => { const home = await createTestTempDirectory("rea-grok-escape-e-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -970,7 +945,6 @@ describe("Grok Build scalar and nested values", () => { it("updates a closed inline table whose last value is a scalar", async () => { const home = await createTestTempDirectory("rea-grok-scalar-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -995,7 +969,6 @@ describe("Grok Build scalar and nested values", () => { it("removes a nested array inside a Grok REA table", async () => { const home = await createTestTempDirectory("rea-grok-nested-array-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = [ @@ -1043,7 +1016,6 @@ describe("Grok Build scalar and nested values", () => { describe("Grok Build newlines and leading marks", () => { it("edits a BOM-prefixed Grok document without losing the first table", async () => { const home = await createTestTempDirectory("rea-grok-bom-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -1081,7 +1053,6 @@ describe("Grok Build newlines and leading marks", () => { it("keeps CRLF string values in a Grok file that also uses LF", async () => { const home = await createTestTempDirectory("rea-grok-crlf-string-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = @@ -1112,7 +1083,6 @@ describe("Grok Build newlines and leading marks", () => { it("keeps LF string values in a Grok file that also uses CRLF", async () => { const home = await createTestTempDirectory("rea-grok-mixed-newline-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -1138,7 +1108,6 @@ describe("Grok Build newlines and leading marks", () => { describe("Grok Build inline comments", () => { it("keeps comments on dotted Grok fields without repeating a comma", async () => { const home = await createTestTempDirectory("rea-grok-dotted-comment-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = [ @@ -1180,7 +1149,6 @@ describe("Grok Build inline comments", () => { it("keeps an inline comment that sits before the Grok REA pair", async () => { const home = await createTestTempDirectory("rea-grok-inline-before-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); const original = @@ -1216,7 +1184,6 @@ describe("Grok Build inline comments", () => { it("keeps inline-table comments and encodes a DEL in an inline value", async () => { const home = await createTestTempDirectory("rea-grok-inline-comment-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -1247,7 +1214,6 @@ describe("Grok Build inline comments", () => { it("keeps comments when uninstall removes the only inline Grok server", async () => { const home = await createTestTempDirectory("rea-grok-inline-empty-"); - roots.push(home); const configPath = join(home, ".grok/config.toml"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -1267,7 +1233,6 @@ describe("Grok Build inline comments", () => { describe("JSON client configuration comments", () => { it("keeps a same-line JSONC comment when uninstall removes Cursor", async () => { const home = await createTestTempDirectory("rea-cursor-jsonc-comment-"); - roots.push(home); const configPath = join(home, ".cursor/mcp.json"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -1289,7 +1254,6 @@ describe("JSON client configuration comments", () => { it("keeps a JSONC comment when a block comment precedes the comma", async () => { const home = await createTestTempDirectory("rea-cursor-jsonc-block-"); - roots.push(home); const configPath = join(home, ".cursor/mcp.json"); await mkdir(dirname(configPath), { recursive: true }); await writeFile( @@ -1314,7 +1278,6 @@ describe("JSON client configuration comments", () => { it("keeps a JSONC comment without swallowing the next server or closers", async () => { const home = await createTestTempDirectory("rea-cursor-jsonc-next-"); - roots.push(home); const configPath = join(home, ".cursor/mcp.json"); await mkdir(dirname(configPath), { recursive: true }); const files = [ @@ -1347,7 +1310,6 @@ describe("JSON client configuration comments", () => { describe("Grok Bot registration", () => { it("does not write or align a data-directory mcp.json", async () => { const home = await createTestTempDirectory("rea-grokbot-"); - roots.push(home); const directory = join(home, ".grokbot"); const configPath = join(directory, "mcp.json"); await mkdir(directory, { recursive: true }); @@ -1388,7 +1350,6 @@ describe("Grok Bot registration", () => { describe("Codex configuration comments", () => { it("keeps Codex comments and value spelling through setup and uninstall", async () => { const home = await createTestTempDirectory("rea-toml-comments-"); - roots.push(home); const configPath = join(home, ".codex/config.toml"); const original = '# Keep this explanation.\nnotify = ["a", "b"]\nliteral = \'C:\\demo\\path\'\n'; @@ -1419,7 +1380,6 @@ describe("managed data purge root validation", () => { "retains cache and state when the managed-root symlink is %s", async (targetState) => { const root = await createTestTempDirectory("rea-uninstall-data-root-"); - roots.push(root); const home = join(root, "home"); const outside = join(root, "unrelated-application"); await mkdir(home); @@ -1458,7 +1418,6 @@ describe("managed data purge root validation", () => { it("retains a managed data root that is not a directory", async () => { const home = await createTestTempDirectory("rea-uninstall-data-file-"); - roots.push(home); const managedRoot = join(home, ".rea"); await writeFile(managedRoot, "unrelated file"); @@ -1477,7 +1436,6 @@ describe("managed data purge root validation", () => { it("purges ordinary managed data through a legitimate home directory link", async () => { const root = await createTestTempDirectory("rea-uninstall-home-link-"); - roots.push(root); const home = join(root, "actual-home"); const linkedHome = join(root, "linked-home"); const managedRoot = join(home, ".rea"); @@ -1514,7 +1472,6 @@ describe("managed data purge root validation", () => { it("reports root inspection failure without removing managed data", async () => { const home = await createTestTempDirectory("rea-uninstall-data-denied-"); - roots.push(home); const managedRoot = join(home, ".rea"); for (const name of ["cache", "state"]) { await mkdir(join(managedRoot, name), { recursive: true }); @@ -1557,7 +1514,6 @@ describe("managed data purge root validation", () => { describe("client configuration filesystem removal", () => { it("uninstalls only owned entries and refuses purge symlinks", async () => { const home = await createTestTempDirectory("rea-uninstall-"); - roots.push(home); const cursor = join(home, ".cursor/mcp.json"); const skill = join( home, @@ -1600,7 +1556,6 @@ describe("client configuration filesystem removal", () => { it("removes emptied skill directories and keeps ones with user files", async () => { const home = await createTestTempDirectory("rea-uninstall-skill-dirs-"); - roots.push(home); const shared = join(home, ".agents/skills/reverse-engineer-anything"); const claude = join(home, ".claude/skills/reverse-engineer-anything"); for (const root of [shared, claude]) { @@ -1633,7 +1588,6 @@ describe("client configuration filesystem removal", () => { "reports an emptied skill directory that cannot be removed", async () => { const home = await createTestTempDirectory("rea-uninstall-skill-perm-"); - roots.push(home); const parent = join(home, ".agents/skills"); const root = join(parent, "reverse-engineer-anything"); await mkdir(join(root, "references"), { recursive: true }); @@ -1664,7 +1618,6 @@ describe("client configuration filesystem removal", () => { describe("uninstall configuration symlink ownership", () => { it("removes an owned entry through a symlink without replacing it", async () => { const home = await createTestTempDirectory("rea-uninstall-symlink-"); - roots.push(home); const configPath = join(home, ".cursor/mcp.json"); const targetPath = join(home, "managed-mcp.json"); const original = JSON.stringify({ @@ -1695,7 +1648,6 @@ describe("uninstall configuration symlink ownership", () => { it("fails before mutation when an uninstall config symlink is dangling", async () => { const home = await createTestTempDirectory("rea-uninstall-symlink-"); - roots.push(home); const configPath = join(home, ".cursor/mcp.json"); await mkdir(dirname(configPath), { recursive: true }); await symlink(join(home, "missing-mcp.json"), configPath); @@ -1719,7 +1671,6 @@ describe("uninstall configuration symlink ownership", () => { describe("uninstall registration ownership preflight", () => { it("removes repeated owned registrations from the table the client reads", async () => { const home = await createTestTempDirectory("rea-uninstall-repeated-owned-"); - roots.push(home); const config = join(home, ".claude.json"); await mkdir(dirname(config), { recursive: true }); const owned = { @@ -1748,7 +1699,6 @@ describe("uninstall registration ownership preflight", () => { const home = await createTestTempDirectory( "rea-uninstall-repeated-foreign-", ); - roots.push(home); const claude = join(home, ".claude.json"); const cursor = join(home, ".cursor/mcp.json"); await mkdir(dirname(claude), { recursive: true }); @@ -1785,7 +1735,6 @@ describe("uninstall registration ownership preflight", () => { const home = await createTestTempDirectory( "rea-uninstall-opencode-fallback-", ); - roots.push(home); const config = join(home, ".config/opencode/opencode.json"); await mkdir(dirname(config), { recursive: true }); const original = JSON.stringify({ @@ -1820,7 +1769,6 @@ describe("layered uninstall outcomes", () => { const home = await createTestTempDirectory( "rea-uninstall-opencode-layers-", ); - roots.push(home); const directory = join(home, ".config/opencode"); const json = join(directory, "opencode.json"); const jsonc = join(directory, "opencode.jsonc"); @@ -1897,7 +1845,6 @@ describe("layered uninstall outcomes", () => { describe("uninstall configuration preflight", () => { it("fails closed on malformed client configuration", async () => { const home = await createTestTempDirectory("rea-uninstall-bad-"); - roots.push(home); const config = join(home, ".cursor/mcp.json"); await mkdir(dirname(config), { recursive: true }); await writeFile(config, "not-json"); @@ -1922,7 +1869,6 @@ describe("uninstall configuration preflight", () => { "names the structural problem in valid JSON %s", async (content, problem) => { const home = await createTestTempDirectory("rea-uninstall-shape-"); - roots.push(home); const config = join(home, ".cursor/mcp.json"); await mkdir(dirname(config), { recursive: true }); await writeFile(config, content); @@ -1948,7 +1894,6 @@ describe("uninstall configuration preflight", () => { it("stops before any removal when one client configuration is malformed", async () => { const home = await createTestTempDirectory("rea-uninstall-stop-"); - roots.push(home); const codex = join(home, ".codex/config.toml"); const cursor = join(home, ".cursor/mcp.json"); const skillMarker = join( @@ -2009,7 +1954,6 @@ describe("uninstall configuration preflight", () => { describe("client configuration changed during uninstall", () => { it("rechecks duplicate ownership after preflight and keeps the changed source", async () => { const home = await createTestTempDirectory("rea-uninstall-repeat-race-"); - roots.push(home); const cursor = join(home, ".cursor/mcp.json"); await mkdir(dirname(cursor), { recursive: true }); const owned = { @@ -2048,7 +1992,6 @@ describe("client configuration changed during uninstall", () => { it("stops the remaining removals when a configuration fails after preflight", async () => { const home = await createTestTempDirectory("rea-uninstall-race-"); - roots.push(home); const codex = join(home, ".codex/config.toml"); const cursor = join(home, ".cursor/mcp.json"); const skill = join( @@ -2203,7 +2146,6 @@ describe("client configuration filesystem failure reporting", () => { it("reports a managed-path removal failure", async () => { const home = await createTestTempDirectory("rea-uninstall-remove-"); - roots.push(home); const skillRoot = join(home, ".agents/skills/reverse-engineer-anything"); await mkdir(skillRoot, { recursive: true }); await writeFile(join(skillRoot, "SKILL.md"), ownedSkillManifest); @@ -2228,7 +2170,6 @@ describe("client configuration filesystem failure reporting", () => { it("uninstalls all managed skill roots, including a custom Claude Code directory", async () => { const home = await createTestTempDirectory("rea-uninstall-skills-"); - roots.push(home); const customClaudeDirectory = join(home, "claude-config"); const rootsToRemove = [ join(home, ".agents/skills/reverse-engineer-anything"), @@ -2263,7 +2204,6 @@ describe("client configuration filesystem failure reporting", () => { describe("skill root client identifier labels", () => { it("labels each retained or removed skill root by its client identifier", async () => { const home = await createTestTempDirectory("rea-uninstall-skill-labels-"); - roots.push(home); const customClaudeDirectory = join(home, "claude-config"); const sharedRoot = join(home, ".agents/skills/reverse-engineer-anything"); const claudeRoot = join( @@ -2312,7 +2252,6 @@ const uninstallFixture = async (): Promise<{ readonly config: string; }> => { const home = await createTestTempDirectory("rea-uninstall-failure-"); - roots.push(home); const config = join(home, ".cursor/mcp.json"); await mkdir(dirname(config), { recursive: true }); await writeFile( diff --git a/tests/boundary/filesystem/setupUnavailableProvider.test.ts b/tests/boundary/filesystem/setupUnavailableProvider.test.ts index af3777577e..d8c5dec59b 100644 --- a/tests/boundary/filesystem/setupUnavailableProvider.test.ts +++ b/tests/boundary/filesystem/setupUnavailableProvider.test.ts @@ -1,6 +1,6 @@ -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; import { createDoctorHostFixture } from "../../../src/application/Doctor.fixture.js"; import { systemSetupHost } from "../../../src/application/SetupHost.js"; @@ -10,17 +10,9 @@ import { supportedClients } from "../../../src/application/SupportedClients.js"; import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -const roots: string[] = []; -afterEach(async () => { - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); -}); - describe("setup with configured but unavailable providers", () => { it("preserves Hopper settings and makes the second setup perform no writes", async () => { const root = await createTestTempDirectory("rea-setup-unavailable-"); - roots.push(root); const client = supportedClients(root, "darwin", {}).find( (candidate) => candidate.name === "cursor", ); diff --git a/tests/boundary/inspector/javascriptRuntimeTargetLocations.test.ts b/tests/boundary/inspector/javascriptRuntimeTargetLocations.test.ts index 56013bcec2..b5b8a9877a 100644 --- a/tests/boundary/inspector/javascriptRuntimeTargetLocations.test.ts +++ b/tests/boundary/inspector/javascriptRuntimeTargetLocations.test.ts @@ -1,23 +1,15 @@ -import { rm, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; -import { afterEach, expect, test } from "vitest"; +import { expect, test } from "vitest"; import { V8InspectorProvider } from "../../../src/inspector/V8InspectorProvider.js"; import { startFakeV8Inspector } from "../../fixtures/inspector/fakeV8Inspector.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -const roots: string[] = []; -afterEach(async () => { - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); -}); - async function entryFile(name: string): Promise { const root = await createTestTempDirectory("rea-node-discovery-"); - roots.push(root); const entry = join(root, name); await writeFile(entry, "setInterval(() => {}, 1000);\n"); return entry; diff --git a/tests/boundary/mcp/bundleComparisonMcp.test.ts b/tests/boundary/mcp/bundleComparisonMcp.test.ts index cbe44655d5..a7e890e99b 100644 --- a/tests/boundary/mcp/bundleComparisonMcp.test.ts +++ b/tests/boundary/mcp/bundleComparisonMcp.test.ts @@ -1,11 +1,11 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { ok as resultOk } from "../../../src/domain/result.js"; -import { rm, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -24,13 +24,6 @@ import { import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; -const roots: string[] = []; -afterEach(async () => { - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); -}); - const sourceEvidence = (label: string) => createEvidence( undefined, @@ -47,7 +40,6 @@ const sourceEvidence = (label: string) => describe("bundle comparison MCP semantic status", () => { it("keeps display-only metadata out of deltas and preserves semantic changes", async () => { const root = await createTestTempDirectory("rea-bundle-metadata-mcp-"); - roots.push(root); const leftPath = join(root, "left.json"); const displayPath = join(root, "display.json"); const changedPath = join(root, "changed.json"); @@ -147,7 +139,6 @@ describe("bundle comparison MCP semantic status", () => { it("requires absolute bundle paths without changing server-cwd semantics", async () => { const root = await createTestTempDirectory("rea-bundle-relative-mcp-"); - roots.push(root); const encoded = serializeEvidenceBundle(createEvidenceBundle([])); await Promise.all([ writeFile(join(root, "left.json"), encoded), @@ -259,7 +250,6 @@ it("requires absolute bundle paths without changing server-cwd semantics", async describe("bundle comparison MCP integration", () => { it("reads caller-selected bundle files and returns complete Evidence", async () => { const root = await createTestTempDirectory("rea-bundle-mcp-"); - roots.push(root); const leftPath = join(root, "left.json"); const rightPath = join(root, "right.json"); const leftRecord = sourceEvidence("left"); @@ -302,7 +292,6 @@ describe("bundle comparison MCP integration", () => { it("reads a bundle path outside any configured roots", async () => { const root = await createTestTempDirectory("rea-bundle-root-"); const outside = await createTestTempDirectory("rea-bundle-outside-"); - roots.push(root, outside); const leftPath = join(root, "left.json"); const rightPath = join(outside, "right.json"); const encoded = serializeEvidenceBundle(createEvidenceBundle([])); @@ -330,7 +319,6 @@ describe("bundle comparison MCP integration", () => { it("explains that a single Evidence record is not a bundle", async () => { const root = await createTestTempDirectory("rea-bundle-record-"); - roots.push(root); const record = sourceEvidence("single"); const recordPath = join(root, "record.json"); const bundlePath = join(root, "bundle.json"); @@ -439,7 +427,6 @@ describe("bundle comparison record selections", () => { const root = await createTestTempDirectory( "rea-bundle-pairs-invalid-mcp-", ); - roots.push(root); const left = [sourceEvidence("left-1"), sourceEvidence("left-2")]; const right = [sourceEvidence("right-1"), sourceEvidence("right-2")]; const leftPath = join(root, "left.json"); diff --git a/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts b/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts index 13660b944a..db7d3ec8f9 100644 --- a/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts +++ b/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts @@ -1,25 +1,16 @@ import { createHash } from "node:crypto"; -import { readFile, rm, writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { createGhidraTargetSnapshot } from "../../../../src/ghidra/GhidraTargetSnapshot.js"; -const roots: string[] = []; - -afterEach(async () => { - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); -}); - describe("Ghidra target snapshot", () => { it("uses the selected installation platform instead of the ambient host", async () => { const root = await createTestTempDirectory("rea-ghidra-platform-"); - roots.push(root); const source = join(root, "source.exe"); const bytes = Buffer.from("platform fixture"); const sha256 = createHash("sha256").update(bytes).digest("hex"); diff --git a/tests/boundary/providers/ghidra/ghidraTransport.test.ts b/tests/boundary/providers/ghidra/ghidraTransport.test.ts index fa703d7d26..01df06227b 100644 --- a/tests/boundary/providers/ghidra/ghidraTransport.test.ts +++ b/tests/boundary/providers/ghidra/ghidraTransport.test.ts @@ -1,4 +1,4 @@ -import { rm, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -12,20 +12,15 @@ import type { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeR import { observeGhidraEndpoint } from "../../../../src/ghidra/GhidraTransport.js"; -const roots: string[] = []; const runtimes: PrivateRuntimeRoot[] = []; afterEach(async () => { await Promise.all(runtimes.splice(0).map((runtime) => runtime.close())); - await Promise.all( - roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), - ); }); describe("Ghidra local transport", () => { it("observes Unix readiness without interpreting endpoint content", async () => { const root = await createTestTempDirectory("rea-ghidra-transport-"); - roots.push(root); const path = join(root, "bridge.sock"); await expect( @@ -39,7 +34,6 @@ describe("Ghidra local transport", () => { it("accepts only an exact IPv4 loopback endpoint record", async () => { const root = await createTestTempDirectory("rea-ghidra-transport-"); - roots.push(root); const runtime = await createGhidraTestRuntime(root); runtimes.push(runtime); const path = await publishGhidraTestEndpoint( @@ -66,7 +60,6 @@ describe("Ghidra local transport", () => { "rejects an invalid or expanded TCP endpoint: $host:$port", async (value) => { const root = await createTestTempDirectory("rea-ghidra-transport-"); - roots.push(root); const runtime = await createGhidraTestRuntime(root); runtimes.push(runtime); const path = await publishGhidraTestEndpoint( diff --git a/tests/composition/idaProvider.test.ts b/tests/composition/idaProvider.test.ts index c32398fed6..f12981cafc 100644 --- a/tests/composition/idaProvider.test.ts +++ b/tests/composition/idaProvider.test.ts @@ -279,7 +279,6 @@ describe("IDA provider selection cancellation", () => { describe("IDA universal Mach-O target identity", () => { const openUniversal = async (registration: Record) => { const root = await createTestTempDirectory("rea-ida-universal-"); - roots.push(root); const input = join(root, "universal"); // x86_64 and arm64 slices share one whole-file digest. await writeFile(input, fat([0x01000007, 0x0100000c])); From aba487eec0303b3392dbea7f33846617158aad3f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 24 Oct 2026 14:31:46 -0700 Subject: [PATCH 21/37] feat(environment): learn creates from every call, with sorted arrays and referenced entities (#7034) * feat(environment): explain values an application derives from other fields Prompt-built environments withheld creates whose responses carry values the application derives deterministically, which the write model kept as constants or could not match across calls: - A composed string is literal text plus parts read from the fields other rules write: a field's value, its slug, or a clock's calendar digits. A create composes a string only when another entity of the type spells it the same way and none spells it otherwise, and never from two sources serving could tell apart; an update composes an unsent change from members stored as sent and unchanged fields that also spell its value before the call. A composed field no longer pins the members it reads. - A trimmed clock format drops trailing fraction zeros, as Go spells nanoseconds. It is learned from the type's values, a later create or a later update, and spreads to the plan's other clocks unless a value shows a fixed width. - A large integer no entity of the type holds, among distinct ones of the same length and kind, is a generated unique value, served as a seeded integer. Co-Authored-By: Claude Opus 5.5 * docs(environment): record the derived-values verification Gates, tests, write plans over 69 recorded captures and served write tasks on the baseline and on this branch. Co-Authored-By: Claude Opus 5.5 * feat(environment): learn a create plan from every call A later create the plan does not take is explained on its own and merged when both calls show the same fields, every field both rule they rule alike, and a field neither rules holds one value: members either call sent are accepted, only those every call sent are required, and an echoed member a call left out takes the value that call stored, kept in the body's defaults. Co-Authored-By: Claude Opus 5.5 * feat(environment): sorted arrays, referenced entities and nested view keys - An echo is `sorted` when the application stored the sent strings in code-point order; serving stores them sorted, and a plain echo and a sorted one agree in update and create plans. - A `reference` is an object field holding fields of the entity a sent member names, as the state holds it. The member is accepted with any value; replay and serving look the entity up, and serving refuses a call naming one the environment does not hold. - A view names in `nested_fields` the keys it always shows of objects it shows with fewer keys than an entity stores; a stored object it never recorded renders with them. The view spelling learner moves to its own module. - Every later create, not only one the plan refuses, is merged into its plan when it agrees, keeping the plan's rule order. Co-Authored-By: Claude Opus 5.5 * fix(environment): keep clock spellings, linked ids and word boundaries honest - A later update's clocks take the plan's spelling, sibling-trimmed clocks included, wherever the call's values fit it, so an update whose later call trims only one of two clocks is served. - A held value that ends its fraction in a zero keeps a create's clock at a fixed width, whatever order the held spellings come in. - A large integer that another field of the created entity also holds is linked, as a repeated token is: it is never unique, and the create is withheld as unexplained-change. - Composed parts are delimited by whole characters through the shared joinsBefore/joinsAfter, so an astral letter joins a match to its word. Co-Authored-By: Claude Opus 5.5 * test(environment): keep a same-width fixed clock out of trimming A held .123456780 beside a held .12345678 keeps the create's clock at nine digits in either order; the cases fail on 8ce1127b5. Co-Authored-By: Claude Opus 5.5 * docs(environment): record the create-calls verification Gates, tests, write plans over 69 recorded captures and served write tasks on the derived-values baseline and on this branch. Co-Authored-By: Claude Opus 5.5 * docs(environment): note the create-calls census on current main Co-Authored-By: Claude Opus 5.5 * fix(environment): check composed parts and clock fits before they shape a plan - A composed string whose part has nothing to spell refuses the create or update through the apply Result instead of keeping a stale value. - A composition learned on a later call is rebuilt from each earlier call's fields, so an earlier call that left the value otherwise contradicts it. - An update adopts its fitted clock spellings only after the unknown-entity and contradiction checks, so a call on an entity an unmodelled write left unknown no longer trims a served plan's clock. - Sibling trimming on a later create also checks that call's fields, so a sibling clock it shows with a trailing zero stays fixed-width. Co-Authored-By: Claude Opus 5.5 * test(environment): split the derived-values view tests by rule Co-Authored-By: Claude Opus 5.5 * fix(environment): keep fixed-width, zone and generated-integer evidence through a plan - A plan keeps the first value that showed each clock at a fixed width, from held entities and every accepted create or update. No later fit or sibling trim widens that field, and a later call that shows it shorter is withheld. - Two stamped clocks are one composed source only when they share a zone offset. A held entity that holds the target field as anything the parts do not spell, a non-string included, vetoes the composition. The token index is built once, and only when a string is left to compose. - A generated integer that another field repeats as a number at any depth is linked. Compare treats two integers of one JSON kind and length under a unique rule as generated values. Co-Authored-By: Claude Opus 5.5 * fix(environment): keep optional references across create calls and in answers - A reference a later create added stays while further creates leave its member out, and agrees with a later call that shows it again. - A create's answer shows optional reference and echo fields only when the call wrote them, whichever recorded call the answer came from. - An update that re-sends the id of the entity a field already repeats keeps the plan's reference instead of pinning the member. - An id naming entities of one type in several collections teaches no reference, and serving or replay refuses a call that names one. - An echo is not promoted to sorted once a call stored its strings out of code-point order as sent. - Nested view keys compare as sets and keep the first recording's order. Co-Authored-By: Claude Opus 5.5 * test(setup): share registration prewarm host preparation * fix(environment): retain write evidence and match trimmed response clocks * fix(environment): withhold creates whose generated integer another string repeats A unique integer whose digits show in another top-level string that no template or composed rule rebuilds from it (an echoed serialText, or a token like folder-) withholds the create as unexplained-change, since serving would seed the integer and keep the string. A templated copy is still served. Co-Authored-By: Claude Opus 5.5 * fix(environment): respect earlier clock widths and left-out members in later creates - The create merge widens a clock to trimmed only against the value that showed its width: a held entity or accepted call at a fixed width keeps it untrimmed. - A later create that left out a member the plan fills, with a default or a reference's absence, while its observation shows that field otherwise is unexplained-change, so the plan cannot serve the stale value. Co-Authored-By: Claude Opus 5.5 * fix(environment): hold later creates and compare to the composed and integer forms - A later create whose response spells a composed field otherwise than the plan rebuilds it is withheld as unexplained-change. - A unique field learned from a generated integer accepts only an integer of the same length and kind from a later create, as serving seeds one. - Compare counts a composed field as equal when each side's response spells it from its own fields, which compare under their own rules. Co-Authored-By: Claude Opus 5.5 * fix(environment): preserve composed parts and unique generator families * fix(environment): reject contradictory create references --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/agent-environments.md | 9 +- .../write-create-calls/plan.md | 12 + .../write-create-calls/spec.md | 48 ++++ .../write-create-calls/tasks.md | 10 + .../write-create-calls/verification.md | 61 +++++ docs/web-network-captures.md | 28 +- .../environment/environmentStateBulkAnswer.ts | 2 + .../environmentStateDerivedAnswer.test.ts | 98 +++++++ .../environmentStateWriteAnswer.ts | 18 +- .../EnvironmentStateWriteSimulation.ts | 1 + .../environmentStateCreateCalls.test.ts | 217 ++++++++++++++++ .../analysisView/environmentStateGate.ts | 81 +----- .../analysisView/environmentStateRules.ts | 27 +- .../analysisView/environmentStateView.ts | 24 +- .../environmentStateViewSpellings.ts | 140 ++++++++++ .../environmentStateWriteApply.ts | 62 ++++- .../environmentStateWriteCalls.ts | 12 +- .../environmentStateWriteChanges.ts | 25 +- .../environmentStateWriteCopies.ts | 3 +- .../environmentStateWriteCreate.ts | 178 ++++++++++--- .../environmentStateWriteCreateCalls.ts | 241 ++++++++++++++++++ .../environmentStateWriteEffects.ts | 30 ++- .../environmentStateWriteExplain.ts | 57 ++++- .../environmentStateWriteReferences.test.ts | 186 ++++++++++++++ .../environmentStateWriteReferences.ts | 99 +++++++ .../environmentStateWriteRules.ts | 46 +++- .../environmentStateWriteValues.ts | 23 ++ .../mcp/environmentServeWritesMcp.test.ts | 38 +++ .../fixtures/environmentStateDerivedValues.ts | 224 ++++++++++++++++ 29 files changed, 1844 insertions(+), 156 deletions(-) create mode 100644 docs/specs/agent-environments/write-create-calls/plan.md create mode 100644 docs/specs/agent-environments/write-create-calls/spec.md create mode 100644 docs/specs/agent-environments/write-create-calls/tasks.md create mode 100644 docs/specs/agent-environments/write-create-calls/verification.md create mode 100644 src/domain/analysisView/environmentStateCreateCalls.test.ts create mode 100644 src/domain/analysisView/environmentStateViewSpellings.ts create mode 100644 src/domain/analysisView/environmentStateWriteCreateCalls.ts create mode 100644 src/domain/analysisView/environmentStateWriteReferences.test.ts create mode 100644 src/domain/analysisView/environmentStateWriteReferences.ts diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 4b812f9659..7b5fbbac00 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -683,7 +683,12 @@ A view that names a field in `sorted_arrays` lists that string array sorted, as a read-back showed the application does, while the state keeps it in the order it was written, so a write's own answer echoes it. If a later counted read contradicts that ordering, the view stays recorded and its earlier reads no -longer confirm served writes. Members a view names in +longer confirm served writes. A view that names a field in `nested_fields` +always showed that field's objects with the same keys, in any order, and fewer +than an entity stores, such +as a list showing a parent without the parent's own nested type; a stored +object it never recorded, such as a created entity's parent, is rendered with +those keys. Members a view names in `per_response_members`, such as a request id each response mints, keep the value its recording carried. A list's `counters` and `query.totals` are top-level members or members one level down in a paging object @@ -721,7 +726,7 @@ every MCP session and `data/get` then see: | Effect | Call | | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. | -| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, unique and renewed changes the plan models. | +| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, reference, unique and renewed changes the plan models. | | `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), with any sent members; it is served only when a recorded call found those fields already set, so a toggle never is. | | `delete` | Removes the entity the path names. | | bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). | diff --git a/docs/specs/agent-environments/write-create-calls/plan.md b/docs/specs/agent-environments/write-create-calls/plan.md new file mode 100644 index 0000000000..f495592330 --- /dev/null +++ b/docs/specs/agent-environments/write-create-calls/plan.md @@ -0,0 +1,12 @@ +# Implementation plan + +1. **Defaults.** `WriteBody.defaults` and its schema; a create's echo of a member a call leaves out takes its default. +2. **Merging.** Explain a later create on its own and merge it: agreeing rules, references one call shows alone, joined + bodies and defaults, the plan's rule order. +3. **Sorted echoes.** Learn `sorted` for creates and updates, store sorted strings when serving, and agree plain and + sorted echoes in update and create plans. +4. **References.** A references module that learns and resolves them; created and updated fields resolve them against + the replayed or live state; the members they read are accepted. +5. **Nested view keys.** The gate's view spelling learner, moved to its own module, notes object keys; rendering + applies them. +6. **Schema, tests and docs.** diff --git a/docs/specs/agent-environments/write-create-calls/spec.md b/docs/specs/agent-environments/write-create-calls/spec.md new file mode 100644 index 0000000000..b84fc52a56 --- /dev/null +++ b/docs/specs/agent-environments/write-create-calls/spec.md @@ -0,0 +1,48 @@ +# Learn creates from every call, sorted arrays and referenced entities + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +After the derived-values change, the creates prompt-built environments still withheld failed for four reasons the +write model did not cover: + +- A create plan was learned from the tool's first call alone, so a later call that sent a member the first did not + (Linkding's `shared`), or left one out, was refused even when it agreed with the plan everywhere else. +- Linkding stores `tag_names` sorted, so a call that sent its tags in another order did not echo them, and its create + was withheld or contradicted. +- Homebox's item create answers its parent and type as whole objects (`parent`, `location`, `entityType`) beside the + `parentId` and `entityTypeId` it sent. Kept as constants, they pinned those members to their recorded values. +- Lists show some objects with fewer keys than an entity holds: Homebox lists an item's parent without the parent's + own type, and Keycloak lists a user's `access` with one key. A created entity's object rendered whole, so the list + check failed. + +## Behaviour + +- **Every call.** A later create is explained on its own and merged into the plan when both calls show the same fields, + but for a reference whose member a call without it did not send (one an earlier create added stays), every field + both rule they rule alike, and a field neither rules holds one value. Members either call sent are accepted, only + those every call sent are required, and an echoed member a call left out takes the value that call stored + (`body.defaults`). A clock widens to `trimmed` only when no held entity or earlier call showed it at a fixed width. A + later create that disagrees leaves the plan as it was and is judged as before; one that left out a member the plan + fills while showing that field otherwise is `unexplained-change`. Merged rules keep the plan's order. +- **Sorted arrays.** An echo is `sorted` when the application stored the sent strings in code-point order rather than + as sent; serving stores them sorted. A plain echo and a sorted one agree, and the plan keeps the sorted one, since a + call whose array was already in order cannot show it; once a call stored strings out of order as sent, they disagree. +- **References.** A created or updated object field that holds fields of the entity a sent member names, as the state + holds it, is a `reference` (member, type, fields). Several members naming such entities resolve to the one named like + the field. The member is accepted with any value. Replay and serving look the entity up in the state they hold; + serving refuses a call naming one the environment does not hold or holds in more than one collection, and the field + is absent, from the entity and the create's answer, when the call does not send the member. An update that sends the + member naming the entity the field already repeats keeps a reference an earlier call showed. +- **Nested view keys.** A view that always shows a field's objects with the same keys, in any order, and fewer than + an entity stores, names them in `nested_fields` in the first recording's order, and a stored object it never + recorded renders with those keys. + +Plans and views that need none of this are unchanged. + +## Acceptance + +- Linkding's prompt-recorded bookmark create whose read-back lists tags sorted is served, and Homebox's item create is + explained without pinning its parent or type. +- No write tool of the 69 recorded captures goes from served to withheld. diff --git a/docs/specs/agent-environments/write-create-calls/tasks.md b/docs/specs/agent-environments/write-create-calls/tasks.md new file mode 100644 index 0000000000..136bd78fde --- /dev/null +++ b/docs/specs/agent-environments/write-create-calls/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] Merge later create calls into the plan, with defaults for members a call leaves out. +- [x] Learn and serve sorted echoes. +- [x] Learn, replay and serve references to held entities. +- [x] Learn the keys views show of nested objects and render written objects with them. +- [x] Add view, serving and unit tests. +- [x] Document the rules and view member. +- [x] Measure the recorded captures and served write tasks. +- [x] Run static, formatting, documentation and test gates. diff --git a/docs/specs/agent-environments/write-create-calls/verification.md b/docs/specs/agent-environments/write-create-calls/verification.md new file mode 100644 index 0000000000..baa0fd4e28 --- /dev/null +++ b/docs/specs/agent-environments/write-create-calls/verification.md @@ -0,0 +1,61 @@ +# Verification + +Executed on macOS with Node v24.22.1 at `043053d1` (main `79845063`, the derived-values branch `8ce1127b` and this +branch). Recorded captures and served write tasks were measured with a local, unpushed build of this branch merged with +the scenario-timing branch (#6967), as the derived-values branch was; its baseline is the derived-values branch with the +same merge. + +## Gates + +- `npm run -s typecheck`, `lint`, `format:check`, `docs:check`, `mcp-catalog:check`, `verify:test-discovery`, + `scan:todos` and `npx knip`: passed. +- `npx vitest run --project domain --project services --project adapters`: 17,258 passed, 25 skipped, 1 failed ("stops + Inspector target authorization after the current stat settles"), which fails the same way on main in this environment. +- `npm run test:focused --` the environment build, export and serve CLI boundaries, the analysis-view CLI and MCP + boundaries, the environment serve, served-writes and bundle MCP boundaries, MCP schema validation and output schema + constraints: 10 files, 82 passed. + +## Tests + +- View: a later create sending a member the first did not, or leaving one out, widens the plan with the member's default; + a later create storing a member otherwise than sent stays unexplained; sent tags stored sorted give a sorted echo; a + create embedding the shelf its `shelf_id` names gives a reference, and the list that shows the shelf by id and name + learns `nested_fields`. +- Serving: a create leaving the member out stores its default and one sending it stores it; a create embeds the named + shelf as the environment holds it, lists it with the listed keys, and is refused for a shelf it does not hold; sent + tags are stored sorted. +- Unit: references (the member named like the field among several, too few keys, a disagreeing value, an unheld entity) + and merges (a constant that differs, a member one fixes and the other accepts, a reference one call shows alone). + +## Write plans over recorded captures + +The 69 captures the derived-values branch measured: 50 published, 18 Claude Code recorded from a prompt, and one timed +Grafana recording. After merging main `ed79bf93` (which bounds create learning), the census was run again against main +with the scenario-timing branch: 124 served there, 132 with both branches, no write going from served to withheld; the +columns below are the measurements before that merge, whose totals it reproduces. + +| Captures | Write plans | Served before both branches | Derived values | This branch | +| ------------------------- | ----------- | --------------------------- | -------------- | ----------- | +| 50 published | 187 | 68 | 68 | 70 | +| 19 recorded from a prompt | 120 | 56 | 61 | 62 | + +- No write goes from served to withheld. +- Gains: Keycloak's published user create and update, whose list shows each user's `access` with one key, so a created + user now lists as recorded and its plan's sorted insert checks; and the round-4 Linkding bookmark create, whose tags + the application stores sorted. +- Explained but still withheld: Homebox's item create now merges with the location create, with references for + `parent`, `location` and `entityType` and `parentId` and `entityTypeId` accepted with any value. Its read-back list + answers items only, sorted by name, from a line with no parameter saying so, which the list model does not cover. +- Linkding's create whose later call sends `shared` merges too, and is withheld as `side-effect`: the recording also + added a bookmark through the UI that no recorded write explains. + +## Served write tasks + +The same captures rebuilt by the baseline and by this branch, with `fireworks_ai/gpt-oss-120b` and +`fireworks_ai/glm-5p3-flash`. The tasks are new; each checks the result from the environment's state, including that +nothing else changed. + +| Capture | Tasks | Baseline (gpt-oss, GLM) | This branch (gpt-oss, GLM) | +| --------------------------- | ------------------------------------------ | ----------------------- | -------------------------- | +| Linkding, recorded round 4 | 2 bookmark creates, one with two tags | 0/2, 0/2 | 2/2, 2/2 | +| Keycloak, published capture | create a user, change another user's email | 0/2, 0/2 | 2/2, 2/2 | diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 7459535193..3498b99418 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -737,14 +737,34 @@ request body must be a usable JSON object (as response bodies are usable), and the written entity must show in that call's response or in a later read before the next write. Each field of a created entity gets the first rule that explains it: generated `identity` (a sequence, UUID or token, by the recorded -value's shape), `echo` of a body member, `copy` of a body string of 8 or more -characters, a `path` value, a `slug` of a body member, a `clock` value within 10 +value's shape), `echo` of a body member (`sorted` when the application stored +its strings in code-point order rather than as sent), `copy` of a body string of +8 or more characters, a `path` value, a `slug` of a body member, a `clock` value within 10 minutes of the request in its recorded format, a `sequence` one past the largest value of its collection or type, a `unique` UUID, opaque token or integer of 10 or more digits that no entity of the type holds (where every one holds a distinct integer of the same length, as time-based ids do), a `nested-clock` object whose clocks the request stamped, a `template` around -a generated value, or a `composed` string; other fields keep the observed value. +a generated value, a `reference` to an entity, or a `composed` string; other +fields keep the observed value. A `reference` is an object holding fields of the +entity a sent member names, as the state holds it (a parent beside `parentId`): +the member is accepted with any value, serving looks the entity up and refuses a +call naming one the environment does not hold or holds in more than one +collection, and the field is absent, from the entity and the create's answer, +when the call does not send the member. An id naming entities of one type in +several collections teaches no reference. Each later create widens its plan +when, explained on its own, it agrees with it: both calls show the same fields +but for a reference whose member a call without it did not send (one an earlier +create added stays), every field both rule they rule alike (an echo one showed +sorted stays sorted, unless a call stored its strings out of order as sent), +and a field neither rules holds one value. Members either call sent are accepted, only those every call sent are +required, and an echoed member a call left out takes the value that call stored +(`body.defaults`). A clock rule widens to `trimmed` only when no held entity or +earlier call showed that clock at a fixed width. A later create that disagrees +leaves the plan as it was. A create whose observed reference differs from the +entity the plan rebuilds, whether or not it sent the reference member, is +`unexplained-change`. So is a create that left out an echoed member yet shows a +value different from the learned default. A `composed` string is literal text and parts read from the fields other rules write: a field's value, its slug, or a clock's calendar digits (`YYYYMMDDhhmmss`, `YYYYMMDD`, `YYYY-MM-DD` or `YYYY/MM/DD`, in the clock's @@ -763,7 +783,7 @@ recorded value (`body.fixed`) and stores the value the call that showed it recorded (the `value` of its `constant` rule), and a field it changed without sending must be a clock, a nested clock, an increment, a slug, a fresh `unique` value, a `stamp` of the acting user, a value `derived` from the one member it -stores nowhere, a `composed` string whose parts read members it stored as sent +stores nowhere, a `reference` to the entity a sent member names (kept by a call that names the entity the field already repeats), a `composed` string whose parts read members it stored as sent or fields it left as they were and spell the value before the call too, or for an item action a `set` of a scalar to the value it showed, or the plan is `unexplained-change`; an increment, slug, stamp, derived diff --git a/src/application/environment/environmentStateBulkAnswer.ts b/src/application/environment/environmentStateBulkAnswer.ts index 33b01e9bf4..8ce6d25434 100644 --- a/src/application/environment/environmentStateBulkAnswer.ts +++ b/src/application/environment/environmentStateBulkAnswer.ts @@ -32,6 +32,7 @@ import { observedEntity, pathValues, unloadable, + heldEntity, updateTarget, writeAnswer, type WriteRequest, @@ -131,6 +132,7 @@ const writtenTargets = ( { before: entity.fields, observed }, call, values.forTarget(updateTarget(serving, entity, observed)), + heldEntity(serving), ); if (!fields.ok) return err(refused(tool, fields.error)); const field = unloadableMember(fields.value); diff --git a/src/application/environment/environmentStateDerivedAnswer.test.ts b/src/application/environment/environmentStateDerivedAnswer.test.ts index 9ec2dca6c8..74817611e6 100644 --- a/src/application/environment/environmentStateDerivedAnswer.test.ts +++ b/src/application/environment/environmentStateDerivedAnswer.test.ts @@ -2,8 +2,13 @@ import { describe, expect, it } from "vitest"; import { BOOKMARK_ROWS, + FILED_ROWS, FOLDER_ROWS, TAG_ROWS, + TAGGED_ROWS, + optionallyFiledRows, + referencedCreateRows, + shelvedRows, } from "../../../tests/fixtures/environmentStateDerivedValues.js"; import { servedFrom } from "../../../tests/fixtures/environmentStateServing.js"; import { @@ -82,3 +87,96 @@ describe("served values an application derives from others", () => { ).toMatchObject({ title: "Moved Folder", url: renamed["url"] }); }); }); + +describe("served creates learned from every call", () => { + it("stores a member a create leaves out at the default a recorded call showed, and a sent one as sent", () => { + const books = servedFrom( + shelvedRows( + { name: "Third Book" }, + { name: "Fourth Book", shelf: "rare" }, + ), + ); + expect( + objectOf(books.answer("post_books", { body: { name: "Plain" } }).body), + ).toMatchObject({ name: "Plain", shelf: "general" }); + expect( + objectOf( + books.answer("post_books", { body: { name: "Kept", shelf: "attic" } }) + .body, + ), + ).toMatchObject({ name: "Kept", shelf: "attic" }); + }); +}); + +describe("served objects that repeat a named entity, and sorted arrays", () => { + it("embeds the shelf a create names as the environment holds it, lists it with the listed keys, and refuses an unknown shelf", () => { + const books = servedFrom(FILED_ROWS); + const created = objectOf( + books.answer("post_books", { body: { name: "Pilot Guide", shelf_id: 2 } }) + .body, + ); + expect(created["shelf"]).toEqual({ + id: 2, + name: "Map Room", + floor: "upper", + }); + const listed = books.answer("get_books").body as JsonValue[]; + expect(listed.at(-1)).toMatchObject({ + name: "Pilot Guide", + shelf: { id: 2, name: "Map Room" }, + }); + expect(JSON.stringify(listed.at(-1))).not.toContain("upper"); + expect( + JSON.stringify( + books.refusal("post_books", { body: { name: "Lost", shelf_id: 9 } }), + ), + ).toContain("names no entity of /api/shelves"); + }); + + it("keeps a contradicted reference out of state writes without changing recording fallback", () => { + const books = servedFrom( + referencedCreateRows({ id: 1, name: "Different Hall", floor: "ground" }), + ); + expect(books.state.writes.get("post_books")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + const before = books.snapshot(); + expect( + books.answer("post_books", { body: { name: "New Book", shelf_id: 1 } }), + ).toMatchObject({ state: { source: "recording" } }); + expect(books.snapshot()).toBe(before); + }); + + it.each([true, false])( + "answers a create with the shelf only when it names one, whichever recorded create was first (shelf first: %s)", + (shelfFirst) => { + const books = servedFrom(optionallyFiledRows(shelfFirst)); + const plain = objectOf( + books.answer("post_books", { body: { name: "Loose Leaf" } }).body, + ); + expect(plain).toMatchObject({ name: "Loose Leaf" }); + expect(plain).not.toHaveProperty("shelf"); + const shelved = objectOf( + books.answer("post_books", { + body: { name: "Pilot Guide", shelf_id: 1 }, + }).body, + ); + expect(shelved).toMatchObject({ + name: "Pilot Guide", + shelf: { id: 1, name: "Main Hall", floor: "ground" }, + }); + }, + ); + + it("stores the tags a create sends in code-point order", () => { + const books = servedFrom(TAGGED_ROWS); + expect( + objectOf( + books.answer("post_books", { + body: { name: "Logbook", tags: ["wind", "ice", "fog"] }, + }).body, + )["tags"], + ).toEqual(["fog", "ice", "wind"]); + }); +}); diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts index 7d803995c2..3d75db50d2 100644 --- a/src/application/environment/environmentStateWriteAnswer.ts +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -1,6 +1,7 @@ import { createdFields, documentBodyProblem, + optionalFields, renewUuids, updatedEntity, updatedFields, @@ -172,6 +173,14 @@ export interface GeneratedTarget { readonly current?: JsonObject; } +/** The fields of each entity of a type with an id the live state holds, which a referencing write repeats. */ +export const heldEntity = + (serving: StateServing) => + (type: string, id: string): JsonObject[] => + serving.store + .ofType(type) + .flatMap((entity) => (entity.id === id ? [entity.fields] : [])); + /** What an update generates values for: the entity it writes, in its collection. */ export const updateTarget = ( serving: StateServing, @@ -335,7 +344,12 @@ export const writeAnswer = ( const body = typeof rendered === "function" ? rendered(recorded) - : writtenResponse(plan.response, recorded, rendered); + : writtenResponse( + plan.response, + recorded, + rendered, + optionalFields(plan), + ); if (!body.ok) return err(refused(tool, body.error)); return stateAnswer(serving, { exchange, exact }, body.value ?? null, state); }; @@ -456,6 +470,7 @@ const answerCreate = ( observed, created, values.forTarget({ name: collection, collection: members, observed }), + heldEntity(serving), ); if (!fields.ok) return err(refused(tool, fields.error)); const field = unloadableMember(fields.value); @@ -544,6 +559,7 @@ const answerUpdate = ( { before: entity.fields, observed }, { ...call, id: entity.id }, values.forTarget(updateTarget(serving, entity, observed)), + heldEntity(serving), ); if (!written.ok) return err(refused(tool, written.error)); // An increment can pass the exact integers. diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index 2a9f21b129..2b524a14fe 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -219,6 +219,7 @@ export class EnvironmentStateWriteSimulation { sequences: new Map(), steady: new Set(), unstamped: new Set(), + kept: new Set(), fixedWidth: new Map(), linked: link.collection, }); diff --git a/src/domain/analysisView/environmentStateCreateCalls.test.ts b/src/domain/analysisView/environmentStateCreateCalls.test.ts new file mode 100644 index 0000000000..198d8bc78a --- /dev/null +++ b/src/domain/analysisView/environmentStateCreateCalls.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, it } from "vitest"; + +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + HELD_FIXED_TAG_ROWS, + TAG_ROWS, + misshelvedRows, + referencedCreateRows, + shelvedRows, +} from "../../../tests/fixtures/environmentStateDerivedValues.js"; +import { environmentStateViewOf } from "../../../tests/fixtures/environmentStateWrites.js"; +import { mergedCreatePlan } from "./environmentStateWriteCreateCalls.js"; +import type { FieldRule } from "./environmentStateWriteRules.js"; + +const createPlan = (rows: readonly TimedCaptureRow[], tool = "post_books") => { + const plan = environmentStateViewOf(timedCaptureEvidence(rows)).writes.find( + (candidate) => candidate.tool === tool, + ); + if (plan === undefined) throw new Error("No create plan"); + return plan; +}; + +const OPTIONAL_SHELF = { + served: true, + body: { + accepted: ["name", "shelf"], + required: ["name"], + defaults: [{ member: "shelf", value: "general" }], + }, + rules: expect.arrayContaining([{ field: "shelf", rule: "echo" }]), +}; + +describe("create plans learned from every call", () => { + it("accepts a member a later create sends, stored as the earlier one's default when left out", () => { + expect( + createPlan( + shelvedRows( + { name: "Third Book" }, + { name: "Fourth Book", shelf: "rare" }, + ), + ), + ).toMatchObject(OPTIONAL_SHELF); + }); + + it("makes a member a later create leaves out optional, with the default that create stored", () => { + expect( + createPlan( + shelvedRows( + { name: "Third Book", shelf: "rare" }, + { name: "Fourth Book" }, + ), + ), + ).toMatchObject(OPTIONAL_SHELF); + }); + + it("withholds creates whose later call stores a left-out member otherwise than the default it learned", () => { + expect(createPlan(misshelvedRows())).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it.each([ + { id: 1, name: "Different Hall", floor: "ground" }, + { id: 2, name: "Map Room", floor: "upper" }, + null, + ])( + "withholds a create whose sent reference contradicts its answer (%j)", + (shelf) => { + expect(createPlan(referencedCreateRows(shelf))).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }, + ); + + it("accepts a later create whose sent reference agrees in a different object key order", () => { + expect( + createPlan( + referencedCreateRows({ floor: "ground", name: "Main Hall", id: 1 }), + ), + ).toMatchObject({ served: true }); + }); + + it("trims no clock a later create shows trimmed after a held entity showed it at a fixed width", () => { + expect(createPlan(TAG_ROWS, "post_tags")).toMatchObject({ served: true }); + const plan = createPlan(HELD_FIXED_TAG_ROWS, "post_tags"); + expect(plan).toMatchObject({ served: false, reason: "unexplained-change" }); + expect(plan.rules).not.toContainEqual( + expect.objectContaining({ + field: "updatedAt", + format: expect.objectContaining({ trimmed: true }), + }), + ); + }); + + it("serves no plan a later create contradicts by storing a member otherwise than sent", () => { + expect( + createPlan( + shelvedRows( + { name: "Third Book" }, + { name: "Fourth Book", shelf: "rare" }, + (sent) => sent.toUpperCase(), + ), + ), + ).toMatchObject({ served: false, reason: "unexplained-change" }); + }); +}); + +describe("merging a later create into a plan", () => { + const body = (accepted: string[], required = accepted) => ({ + accepted, + fixed: [], + required, + identity: null, + kinds: [], + }); + const plan = { + rules: [ + { field: "id", rule: "identity", generator: "sequence" } as const, + { field: "name", rule: "echo" } as const, + ], + body: body(["name"]), + }; + const exemplar = { id: 3, name: "Third", shelf: "general" }; + const later = ( + overrides: Partial[2]>, + ) => ({ + entity: { id: 4, name: "Fourth", shelf: "rare" }, + body: { name: "Fourth", shelf: "rare" }, + rules: [...plan.rules, { field: "shelf", rule: "echo" } as const], + contract: body(["name", "shelf"]), + ...overrides, + }); + + it("refuses a field neither call rules that holds another value, or a member one fixes and the other accepts", () => { + expect( + mergedCreatePlan(plan, exemplar, later({ rules: plan.rules })), + ).toBeUndefined(); + expect( + mergedCreatePlan( + { + ...plan, + body: { ...body(["name"]), fixed: [{ member: "shelf", sent: "x" }] }, + }, + exemplar, + later({}), + ), + ).toBeUndefined(); + }); + + const reference: FieldRule = { + field: "shelf", + rule: "reference", + member: "shelf_id", + type: "/api/shelves", + fields: ["id", "name"], + }; + const shelved = (id: number, name: string) => + later({ + entity: { id, name, shelf: { id: 1, name: "Main" } }, + body: { name, shelf_id: 1 }, + rules: [...plan.rules, reference], + contract: body(["name", "shelf_id"]), + }); + const unshelved = (id: number, name: string, sent: object = {}) => + later({ + entity: { id, name }, + body: { name, ...sent }, + rules: plan.rules, + contract: body(["name", ...Object.keys(sent)]), + }); + const unshelvedExemplar = { id: 3, name: "Third" }; + + it("takes a reference one call alone shows when the other did not send its member", () => { + const merged = mergedCreatePlan( + plan, + unshelvedExemplar, + shelved(4, "Fourth"), + ); + expect(merged).toMatchObject({ + rules: expect.arrayContaining([reference]), + body: { accepted: ["name", "shelf_id"], required: ["name"] }, + }); + }); + + it("keeps a reference an earlier call added through calls that leave it out, and agrees with it when it returns", () => { + const added = mergedCreatePlan( + plan, + unshelvedExemplar, + shelved(4, "Fourth"), + )!; + const omitted = mergedCreatePlan( + added, + unshelvedExemplar, + unshelved(5, "Fifth"), + ); + expect(omitted).toMatchObject({ + rules: expect.arrayContaining([reference]), + body: { accepted: ["name", "shelf_id"], required: ["name"] }, + }); + expect( + mergedCreatePlan(omitted!, unshelvedExemplar, shelved(6, "Sixth")), + ).toMatchObject({ rules: expect.arrayContaining([reference]) }); + // A call that sends the member yet stores no shelf contradicts the reference. + expect( + mergedCreatePlan( + added, + unshelvedExemplar, + unshelved(7, "Seventh", { shelf_id: 1 }), + ), + ).toBeUndefined(); + }); +}); diff --git a/src/domain/analysisView/environmentStateGate.ts b/src/domain/analysisView/environmentStateGate.ts index 29ceceeebd..53321d7464 100644 --- a/src/domain/analysisView/environmentStateGate.ts +++ b/src/domain/analysisView/environmentStateGate.ts @@ -31,7 +31,7 @@ import { type QueryLine, } from "./environmentStateListQuery.js"; import { singleQueryValues } from "../recordedTransportUrl.js"; -import { readClock, type ClockFormat } from "./environmentStateWriteValues.js"; +import { ClockSpellings } from "./environmentStateViewSpellings.js"; import { declaredMembers, mintedEnvelope, @@ -132,85 +132,6 @@ class FieldUnion { } } -/** The spellings a view always gives clock fields where its recordings spell the stored instant otherwise. */ -class ClockSpellings { - /** Each category retains one spelling or a permanent mixed marker, never an expanding format history. */ - readonly #fields = new Map< - string, - { - matched: ClockFormat | null | undefined; - other: ClockFormat | null | undefined; - } - >(); - - constructor(private readonly retain: StateGateContext["retain"]) { - retain(ENTRY_BYTES); - } - - #note( - field: string, - category: "matched" | "other", - format: ClockFormat | null, - ): void { - let facts = this.#fields.get(field); - if (facts === undefined) { - this.retain(ENTRY_BYTES + textBytes(field.length)); - facts = { matched: undefined, other: undefined }; - this.#fields.set(field, facts); - } - const held = facts[category]; - if (held === undefined) { - if (format !== null) - this.retain( - ENTRY_BYTES + - (format.kind === "iso" ? textBytes(format.zone.length) : 0), - ); - facts[category] = format; - } else if (held !== null && !same(held, format)) facts[category] = null; - } - - note(stored: JsonObject, shown: JsonObject): void { - for (const [field, value] of Object.entries(shown)) { - const now = readClock(value); - if (now === undefined) continue; - const was = Object.hasOwn(stored, field) - ? readClock(stored[field]) - : undefined; - if ( - was === undefined || - was.ms !== now.ms || - was.subMillisecond !== now.subMillisecond - ) { - this.#note(field, "other", now.format); - continue; - } - const spelled = same(stored[field], value) ? null : now.format; - this.#note(field, "matched", spelled); - } - } - - /** The view member, when the view spells any field otherwise, and in that spelling only. */ - get view(): { readonly clock_formats?: Record } { - const formats: [string, ClockFormat][] = []; - for (const [field, { matched, other }] of this.#fields) { - if ( - matched === undefined || - matched === null || - other === null || - (other !== undefined && !same(other, matched)) - ) - continue; - if (formats.length === 0) this.retain(ENTRY_BYTES * 2); - // Both the temporary entry and its returned object member remain covered. - this.retain(ENTRY_BYTES * 2 + textBytes(field.length)); - formats.push([field, matched]); - } - return formats.length === 0 - ? {} - : { clock_formats: Object.fromEntries(formats) }; - } -} - const recorded = (plan: StatePlan, reason: RecordedViewReason): StateView => ({ tool: plan.tool.name, shape: "recorded", diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index ccc29110dd..a4d1c0c342 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -120,6 +120,8 @@ interface EntityViewPlan { readonly clock_formats?: Readonly> | undefined; /** Fields whose string arrays the view lists in code-point order, whatever order they were written in. */ readonly sorted_arrays?: string[] | undefined; + /** The keys the view shows of object fields it shows with fewer keys than stored. */ + readonly nested_fields?: Readonly> | undefined; /** Envelope members each response there mints, which checks leave out and serving keeps as recorded. */ readonly per_response_members?: string[] | undefined; } @@ -660,7 +662,7 @@ export const renderStateEntity = ( entity: StateEntity, view: Pick< EntityViewPlan, - "tool" | "fields" | "clock_formats" | "sorted_arrays" + "tool" | "fields" | "clock_formats" | "sorted_arrays" | "nested_fields" >, volatileFields: ReadonlySet, observed?: JsonObject, @@ -669,6 +671,7 @@ export const renderStateEntity = ( entity.view_fields.find((candidate) => candidate.tool === view.tool) ?.fields ?? {}; const formats = view.clock_formats ?? {}; + const nested = view.nested_fields ?? {}; const sorted = new Set(view.sorted_arrays); return Object.fromEntries( view.fields.flatMap((name): [string, JsonValue][] => { @@ -679,16 +682,30 @@ export const renderStateEntity = ( ? own : entity.fields; if (!Object.hasOwn(source, name)) return []; - // A stored value the view never recorded, such as a written one, takes the view's spelling. + // A stored value the view never recorded, such as a written one, takes the view's spelling and keys. const value = - source === entity.fields && Object.hasOwn(formats, name) - ? respellClock(source[name]!, formats[name]!) - : source[name]!; + source !== entity.fields + ? source[name]! + : Object.hasOwn(formats, name) + ? respellClock(source[name]!, formats[name]!) + : Object.hasOwn(nested, name) + ? shownKeys(source[name]!, nested[name]!) + : source[name]!; return [[name, sorted.has(name) ? inSetOrder(value) : value]]; }), ); }; +/** An object with only the keys a view shows of it, in the view's order; any other value as it is. */ +const shownKeys = (value: JsonValue, keys: readonly string[]): JsonValue => + isObject(value) + ? Object.fromEntries( + keys.flatMap((key) => + Object.hasOwn(value, key) ? [[key, value[key]!]] : [], + ), + ) + : value; + /** Whether a value is an array of strings. */ export const isStringArray = (value: JsonValue): value is string[] => Array.isArray(value) && value.every((item) => typeof item === "string"); diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index c01c556b06..62ef9e08a9 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -128,6 +128,12 @@ const entityViewFields = { .describe( "How the view spells clock fields it always spells otherwise than the entities' stored values; a stored value the view never recorded is spelled this way.", ), + nested_fields: z + .record(z.string(), z.array(z.string())) + .optional() + .describe( + "The keys the view shows of object fields it always shows with fewer keys than an entity stores, such as a parent listed without its own nested type; a stored object the view never recorded shows those keys.", + ), sorted_arrays: z .array(z.string()) .optional() @@ -253,7 +259,11 @@ const fieldRuleSchema = z.discriminatedUnion("rule", [ rule: z.literal("held-clock"), format: clockFormatSchema, }), - z.strictObject({ ...ruled, rule: z.literal("echo") }), + z.strictObject({ + ...ruled, + rule: z.literal("echo"), + sorted: z.literal(true).exactOptional(), + }), z.strictObject({ ...ruled, rule: z.literal("copy"), from: z.string() }), z.strictObject({ ...ruled, rule: z.literal("path"), from: z.string() }), z.strictObject({ ...ruled, rule: z.literal("slug"), from: z.string() }), @@ -278,6 +288,13 @@ const fieldRuleSchema = z.discriminatedUnion("rule", [ ) .min(1), }), + z.strictObject({ + ...ruled, + rule: z.literal("reference"), + member: z.string(), + type: z.string(), + fields: z.array(z.string()).min(1), + }), z.strictObject({ ...ruled, rule: z.literal("increment") }), z.strictObject({ ...ruled, @@ -360,10 +377,13 @@ const writePlanSchema = z.strictObject({ }), ), envelope: z.string().optional(), + defaults: z + .array(z.strictObject({ member: z.string(), value: jsonValueSchema })) + .optional(), }) .nullable() .describe( - "Body members the application stored as sent (accepted), members accepted only with their recorded value (fixed), members every call sends (required), the JSON type a later call stored an accepted member as when the observation does not show it (kinds), and the member a call wraps its object in as a one-element array (envelope).", + "Body members the application stored as sent (accepted), members accepted only with their recorded value (fixed), members every call sends (required), the JSON type a later call stored an accepted member as when the observation does not show it (kinds), the member a call wraps its object in as a one-element array (envelope), and the value a create stores for an accepted member a call leaves out, as a call that left it out showed (defaults).", ), rules: z .array(fieldRuleSchema) diff --git a/src/domain/analysisView/environmentStateViewSpellings.ts b/src/domain/analysisView/environmentStateViewSpellings.ts new file mode 100644 index 0000000000..488dc216c8 --- /dev/null +++ b/src/domain/analysisView/environmentStateViewSpellings.ts @@ -0,0 +1,140 @@ +import type { JsonValue } from "../jsonValue.js"; +import { isObject, same, type JsonObject } from "./environmentStateRules.js"; +import { readClock, type ClockFormat } from "./environmentStateWriteValues.js"; +import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; + +/** Retain the bytes a view's learned members keep, as the gate's budget counts them. */ +type Retain = (bytes: number) => void; + +/** Whether two objects show the same keys, in any order. */ +const sameKeys = (left: readonly string[], shown: JsonObject): boolean => + left.length === Object.keys(shown).length && + left.every((key) => Object.hasOwn(shown, key)); + +/** The spellings a view always gives clock fields where its recordings spell the stored instant otherwise. */ +export class ClockSpellings { + /** Each category retains one spelling or a permanent mixed marker, never an expanding format history. */ + readonly #fields = new Map< + string, + { + matched: ClockFormat | null | undefined; + other: ClockFormat | null | undefined; + } + >(); + + /** Per object field, the keys every recording showed in the first one's order, or null once two differ, and whether one showed fewer than stored. */ + readonly #shapes = new Map< + string, + { keys: string[] | null; projects: boolean } + >(); + + constructor(private readonly retain: Retain) { + retain(ENTRY_BYTES); + } + + #shape( + field: string, + shown: JsonObject, + stored: JsonValue | undefined, + ): void { + const keys = Object.keys(shown); + const projects = + isObject(stored) && + Object.keys(stored).some((key) => !Object.hasOwn(shown, key)); + const facts = this.#shapes.get(field); + if (facts === undefined) { + this.retain( + ENTRY_BYTES * 2 + + textBytes(field.length) + + keys.reduce( + (sum, key) => sum + ENTRY_BYTES + textBytes(key.length), + 0, + ), + ); + this.#shapes.set(field, { keys, projects }); + return; + } + if (facts.keys !== null && !sameKeys(facts.keys, shown)) facts.keys = null; + facts.projects ||= projects; + } + + #note( + field: string, + category: "matched" | "other", + format: ClockFormat | null, + ): void { + let facts = this.#fields.get(field); + if (facts === undefined) { + this.retain(ENTRY_BYTES + textBytes(field.length)); + facts = { matched: undefined, other: undefined }; + this.#fields.set(field, facts); + } + const held = facts[category]; + if (held === undefined) { + if (format !== null) + this.retain( + ENTRY_BYTES + + (format.kind === "iso" ? textBytes(format.zone.length) : 0), + ); + facts[category] = format; + } else if (held !== null && !same(held, format)) facts[category] = null; + } + + note(stored: JsonObject, shown: JsonObject): void { + for (const [field, value] of Object.entries(shown)) { + if (isObject(value)) this.#shape(field, value, stored[field]); + const now = readClock(value); + if (now === undefined) continue; + const was = Object.hasOwn(stored, field) + ? readClock(stored[field]) + : undefined; + if ( + was === undefined || + was.ms !== now.ms || + was.subMillisecond !== now.subMillisecond + ) { + this.#note(field, "other", now.format); + continue; + } + const spelled = same(stored[field], value) ? null : now.format; + this.#note(field, "matched", spelled); + } + } + + /** The fields whose objects every recording showed with the same keys, fewer than one stored. */ + get #nested(): { readonly nested_fields?: Record } { + const nested = [...this.#shapes].flatMap(([field, { keys, projects }]) => + keys !== null && projects ? [[field, keys] as const] : [], + ); + return nested.length === 0 + ? {} + : { nested_fields: Object.fromEntries(nested) }; + } + + /** The view members: the spelling of fields it always spells otherwise, and the keys of objects it shows fewer of. */ + get view(): { + readonly clock_formats?: Record; + readonly nested_fields?: Record; + } { + const formats: [string, ClockFormat][] = []; + for (const [field, { matched, other }] of this.#fields) { + if ( + matched === undefined || + matched === null || + other === null || + (other !== undefined && !same(other, matched)) + ) + continue; + if (formats.length === 0) this.retain(ENTRY_BYTES * 2); + // Both the temporary entry and its returned object member remain covered. + this.retain(ENTRY_BYTES * 2 + textBytes(field.length)); + formats.push([field, matched]); + } + return { + ...(formats.length === 0 + ? {} + : { clock_formats: Object.fromEntries(formats) }), + ...this.#nested, + }; + } +} diff --git a/src/domain/analysisView/environmentStateWriteApply.ts b/src/domain/analysisView/environmentStateWriteApply.ts index f980e1138b..c81fa2af04 100644 --- a/src/domain/analysisView/environmentStateWriteApply.ts +++ b/src/domain/analysisView/environmentStateWriteApply.ts @@ -26,6 +26,11 @@ import { } from "./environmentStateWriteStamps.js"; import { composedValue } from "./environmentStateWriteComposed.js"; import { + referencedValue, + type ReferencedEntity, +} from "./environmentStateWriteReferences.js"; +import { + echoed, integerOf, readClock, isUuid, @@ -211,9 +216,12 @@ const ruledValue = ( observed: JsonObject, call: WriteCall, generate: GenerateValue, + referenced?: ReferencedEntity, ): Applied => { const body = call.body ?? {}; switch (rule.rule) { + case "reference": + return referencedValue(rule, body, referenced); case "identity": case "sequence": case "unique": @@ -223,7 +231,11 @@ const ruledValue = ( case "held-clock": return generate(rule); case "echo": - return ok(Object.hasOwn(body, rule.field) ? body[rule.field] : undefined); + return ok( + Object.hasOwn(body, rule.field) + ? echoed(rule, body[rule.field]!) + : undefined, + ); case "copy": { const value = valueAt(body, rule.from); return value === undefined @@ -314,6 +326,7 @@ export const createdFields = ( observed: JsonObject, call: WriteCall, generate: GenerateValue, + referenced?: ReferencedEntity, ): Applied => { if (plan.body === null) return err("The tool has no modelled body."); const problem = bodyProblem(plan.body, call, { @@ -321,11 +334,17 @@ export const createdFields = ( }); if (problem !== undefined) return err(problem); const written: JsonObject = { ...observed }; + const defaults = new Map( + (plan.body.defaults ?? []).map(({ member, value }) => [member, value]), + ); for (const rule of plan.rules) { - const value = ruledValue(rule, observed, call, generate); + const value = ruledValue(rule, observed, call, generate, referenced); if (!value.ok) return value; if (value.value !== undefined) written[rule.field] = value.value; - else if (rule.rule === "echo") delete written[rule.field]; + else if (rule.rule === "echo" && defaults.has(rule.field)) + written[rule.field] = defaults.get(rule.field)!; + else if (rule.rule === "echo" || rule.rule === "reference") + delete written[rule.field]; } const unspelled = derive(plan.rules, observed, written, call.body ?? {}); return unspelled === undefined ? ok(written) : err(unspelled); @@ -353,7 +372,7 @@ const statedValue = ( if (rule.rule === "set") return rule.value; if (rule.rule !== "echo" && rule.rule !== "constant") return undefined; if (!Object.hasOwn(sent, rule.field)) return undefined; - return rule.rule === "echo" ? sent[rule.field] : rule.value; + return rule.rule === "echo" ? echoed(rule, sent[rule.field]!) : rule.value; }; /** The fields of an updated entity: sent echoed members, the constants the plan recorded, generated and derived changes. */ @@ -362,6 +381,7 @@ export const updatedFields = ( entity: { readonly before: JsonObject; readonly observed: JsonObject }, call: WriteCall & { readonly id: string }, generate: GenerateValue, + referenced?: ReferencedEntity, ): Applied => { const { before, observed } = entity; if (plan.body === null) return err("The tool has no modelled body."); @@ -384,7 +404,11 @@ export const updatedFields = ( for (const rule of plan.rules) { const stated = statedValue(rule, sent); if (stated !== undefined) written[rule.field] = stated; - else if (rule.rule === "increment") { + else if (rule.rule === "reference" && Object.hasOwn(sent, rule.member)) { + const value = referencedValue(rule, sent, referenced); + if (!value.ok) return value; + if (value.value !== undefined) written[rule.field] = value.value; + } else if (rule.rule === "increment") { const was = integerOf(before[rule.field]); if (was !== undefined) written[rule.field] = spelledLike(before[rule.field]!, was + 1); @@ -444,11 +468,28 @@ export const writtenDocument = ( return ok(replaceAt(before, pointer, value)); }; -/** A write's answer body from its recorded response: the entity or identity rendered in, nothing, or the recorded body. */ +/** The fields a create stores only when a call sends their member: references and echoes of members not every call sent, with no default. */ +export const optionalFields = ( + plan: Pick, +): string[] => { + if (plan.effect !== "create" || plan.body === null) return []; + const { required, defaults = [] } = plan.body; + return plan.rules.flatMap((rule) => + (rule.rule === "reference" && !required.includes(rule.member)) || + (rule.rule === "echo" && + !required.includes(rule.field) && + !defaults.some(({ member }) => member === rule.field)) + ? [rule.field] + : [], + ); +}; + +/** A write's answer body from its recorded response: the entity or identity rendered in, nothing, or the recorded body; optional fields show only when written. */ export const writtenResponse = ( response: WriteResponse, recorded: JsonValue | undefined, written: { readonly fields: JsonObject | undefined; readonly id: string }, + optional: readonly string[] = [], ): Applied => { if (response.shape === "empty") return ok(undefined); if (response.shape === "recorded" || recorded === undefined) @@ -467,19 +508,24 @@ export const writtenResponse = ( ); if (!isObject(shown) || written.fields === undefined) return err("The recorded response does not carry the entity."); + const held = (field: string) => Object.hasOwn(written.fields!, field); const missing = Object.keys(shown).filter( - (field) => !Object.hasOwn(written.fields!, field), + (field) => !held(field) && !optional.includes(field), ); if (missing.length > 0) return err( `The state holds no complete entity for this response: it lacks ${missing.join(", ")}.`, ); + const fields = [ + ...Object.keys(shown).filter(held), + ...optional.filter((field) => !Object.hasOwn(shown, field) && held(field)), + ]; return ok( replaceAt( recorded, pointer, Object.fromEntries( - Object.keys(shown).map((field) => [field, written.fields![field]!]), + fields.map((field) => [field, written.fields![field]!]), ), ), ); diff --git a/src/domain/analysisView/environmentStateWriteCalls.ts b/src/domain/analysisView/environmentStateWriteCalls.ts index 6b0402f909..c5a691f795 100644 --- a/src/domain/analysisView/environmentStateWriteCalls.ts +++ b/src/domain/analysisView/environmentStateWriteCalls.ts @@ -5,10 +5,11 @@ import { same, type JsonObject } from "./environmentStateRules.js"; import { storedKinds } from "./environmentStateWriteApply.js"; import { composedValue } from "./environmentStateWriteComposed.js"; import type { explainUpdate } from "./environmentStateWriteExplain.js"; -import type { - FieldRule, - StateWritePlan, - WriteBody, +import { + alikeRules, + type FieldRule, + type StateWritePlan, + type WriteBody, } from "./environmentStateWriteRules.js"; import { integerOf, slugOf } from "./environmentStateWriteValues.js"; @@ -186,6 +187,7 @@ export const refinedBody = ( export const conflictsWithPlan = ( plan: Pick, explained: ReturnType, + kept: ReadonlySet, ): boolean => { const body = plan.body!; const fixed = new Map(body.fixed.map((entry) => [entry.member, entry.sent])); @@ -203,7 +205,7 @@ export const conflictsWithPlan = ( (identity !== null && fixed.has(identity)) || explained.rules.some((rule) => { const held = rules.get(rule.field); - return held !== undefined && !same(held, rule); + return held !== undefined && alikeRules(held, rule, kept) === undefined; }) ); }; diff --git a/src/domain/analysisView/environmentStateWriteChanges.ts b/src/domain/analysisView/environmentStateWriteChanges.ts index 1786655e6f..e7336a3854 100644 --- a/src/domain/analysisView/environmentStateWriteChanges.ts +++ b/src/domain/analysisView/environmentStateWriteChanges.ts @@ -1,9 +1,19 @@ import type { JsonValue } from "../jsonValue.js"; -import { isObject, same, type JsonObject } from "./environmentStateRules.js"; +import { + isObject, + same, + type JsonObject, + type StateEntity, +} from "./environmentStateRules.js"; import type { FieldRule, WriteCall } from "./environmentStateWriteRules.js"; -import { integerOf, readClock } from "./environmentStateWriteValues.js"; +import { + integerOf, + readClock, + sortedStrings, +} from "./environmentStateWriteValues.js"; import { freshUuid, renewedRule } from "./environmentStateWriteGenerated.js"; import { composedChange } from "./environmentStateWriteComposed.js"; +import { referenceRule } from "./environmentStateWriteReferences.js"; import { derivedRule, freshToken, @@ -29,6 +39,10 @@ export interface UpdateEvidence { readonly shown?: readonly string[]; /** An item action, whose unsent changes take the values it showed. */ readonly action?: boolean; + /** The entities of any type the state holds under an id, which an object the update rewrote may repeat. */ + readonly named?: ((id: string) => readonly StateEntity[]) | undefined; + /** The references earlier calls showed, which a call naming the entity a field already repeats keeps. */ + readonly references?: readonly Extract[]; } /** The rule a field the update changed without sending follows, if any; `slugs` maps each echoed member's slug to the first such member. */ @@ -48,6 +62,11 @@ export const changeRule = ( if (freshToken(evidence, field)) return { field, rule: "unique" }; const nested = nestedClockRule(field, { before, after }, evidence); if (nested !== undefined) return nested; + const reference = + evidence.named === undefined + ? undefined + : referenceRule(field, after, evidence.call.body, evidence.named); + if (reference !== undefined) return reference; // Child rows only: a single UUID that changed is a reference or a generated value, judged above. const renewed = Array.isArray(before) || isObject(before) @@ -135,6 +154,8 @@ export const sentRule = ( }; if (same(evidence.after[member], sent)) return { field: member, rule: "echo" }; + if (same(evidence.after[member], sortedStrings(sent))) + return { field: member, rule: "echo", sorted: true }; return freshUuid(evidence, member) ? { field: member, rule: "unique" } : { field: member, rule: "constant", value: evidence.after[member]! }; diff --git a/src/domain/analysisView/environmentStateWriteCopies.ts b/src/domain/analysisView/environmentStateWriteCopies.ts index d920ffd124..f4a736c9f1 100644 --- a/src/domain/analysisView/environmentStateWriteCopies.ts +++ b/src/domain/analysisView/environmentStateWriteCopies.ts @@ -34,7 +34,7 @@ export const templateRule = ( return undefined; }; -/** Body members the entity stores as sent, under their own name (echo) or only another (a copy of the whole member). */ +/** Body members the entity stores as sent, under their own name (echo) or only another (a copy of the whole member), or names an entity by (a reference). */ export const storedMembers = ( rules: readonly FieldRule[], entity: JsonObject, @@ -42,6 +42,7 @@ export const storedMembers = ( new Set( rules.flatMap((rule) => { if (rule.rule === "echo") return [rule.field]; + if (rule.rule === "reference") return [rule.member]; const tokens = rule.rule === "copy" ? pointerTokens(rule.from) : []; // A member also stored otherwise under its own name is not stored as sent. return tokens.length === 1 && !Object.hasOwn(entity, tokens[0]!) diff --git a/src/domain/analysisView/environmentStateWriteCreate.ts b/src/domain/analysisView/environmentStateWriteCreate.ts index 78c1f27c4e..b58442780c 100644 --- a/src/domain/analysisView/environmentStateWriteCreate.ts +++ b/src/domain/analysisView/environmentStateWriteCreate.ts @@ -5,6 +5,7 @@ import { isObject, isParam, normalizedPath, + same, scalarText, segmentsOf, selectedCollection, @@ -35,6 +36,7 @@ import { callOf, clockEvidence, fail, + referencedIn, itemReads, refuse, requestBodyPlan, @@ -62,6 +64,7 @@ import { type Observed, } from "./environmentStateWriteObserve.js"; import { + keptOrders, pathValuesOf, siblingTrimmedClocks, type FieldRule, @@ -71,6 +74,7 @@ import { fittedClockFormat, noteFixedWidths, } from "./environmentStateWriteClockFormats.js"; +import { mergedCreatePlan } from "./environmentStateWriteCreateCalls.js"; import { containsDelimited, generatedIntegerDigits, @@ -183,40 +187,103 @@ const strandedInteger = ( ); }); -/** Build a create plan from its exemplar's observation. */ -const buildCreate = ( +/** Whether a created field no rule explains holds a generated value another field links to, or a unique integer another string strands, which serving cannot keep consistent. */ +const linkedUnexplained = ( + rules: readonly FieldRule[], + entity: JsonObject, + held: readonly JsonObject[], +): boolean => { + const explained = new Set(rules.map((rule) => rule.field)); + return ( + Object.keys(entity).some( + (field) => !explained.has(field) && linkedGenerated(field, entity, held), + ) || strandedInteger(rules, entity) + ); +}; + +/** Widen a create plan by a later call, when that call explained on its own agrees with it; whether it did. */ +const refineCreate = ( context: EffectContext, draft: WriteDraft, step: WriteStep, - evidence: { - readonly observed: Observed; - readonly call: WriteCall & { readonly body: JsonObject }; - readonly collection: string; - readonly envelope: string | undefined; - }, + evidence: CreateCallEvidence, ): boolean => { + const { explained, held } = explainedCall(context, draft, step, evidence); + for (const member of keptOrders(evidence.call.body, evidence.observed.object)) + draft.kept.add(member); + if ( + (explained.tied && draft.plan.collection === null) || + linkedUnexplained(explained.rules, evidence.observed.object, held) + ) + return false; + const merged = mergedCreatePlan( + { rules: draft.plan.rules, body: draft.plan.body! }, + draft.observed!, + { + entity: evidence.observed.object, + body: evidence.call.body, + rules: explained.rules, + contract: { + ...explained.body, + ...(evidence.envelope === undefined + ? {} + : { envelope: evidence.envelope }), + }, + }, + { kept: draft.kept, clockOf: (field) => clockEvidence(draft, field) }, + ); + if (merged === undefined) return false; + retainFixed(context, merged.body.fixed.slice(draft.plan.body!.fixed.length)); + draft.plan.rules = merged.rules; + draft.plan.body = merged.body; + return true; +}; + +type CreateCallEvidence = { + readonly observed: Observed; + readonly call: WriteCall & { readonly body: JsonObject }; + readonly collection: string; + readonly envelope: string | undefined; +}; + +/** Explain one recorded create on its own, against the entities its collection and type held before it. */ +const explainedCall = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, + evidence: CreateCallEvidence, +) => { const type = draft.plan.target!; - const idField = context.idField(type); const members = fieldsOf(context.state.members(type, evidence.collection)); const held = fieldsOf(context.state.ofType(type)); const explained = explainCreate({ entity: evidence.observed.object, call: evidence.call, - idField, + idField: context.idField(type), startMs: startOf(context, step), collection: members, type: held, named: (id) => context.state.named(id), }); - const explainedFields = new Set(explained.rules.map((rule) => rule.field)); - if ( - Object.keys(evidence.observed.object).some( - (field) => - !explainedFields.has(field) && - linkedGenerated(field, evidence.observed.object, held), - ) || - strandedInteger(explained.rules, evidence.observed.object) - ) { + return { explained, members, held }; +}; + +/** Build a create plan from its exemplar's observation. */ +const buildCreate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, + evidence: CreateCallEvidence, +): boolean => { + const type = draft.plan.target!; + const idField = context.idField(type); + const { explained, members, held } = explainedCall( + context, + draft, + step, + evidence, + ); + if (linkedUnexplained(explained.rules, evidence.observed.object, held)) { refuse(draft, step, "unexplained-change"); return false; } @@ -246,6 +313,8 @@ const buildCreate = ( }, ); retainFixed(context, explained.body.fixed); + for (const member of keptOrders(evidence.call.body, evidence.observed.object)) + draft.kept.add(member); Object.assign(draft.plan, { exemplar_ordinal: step.record.ordinal, observation: { @@ -321,6 +390,30 @@ const followsSequences = ( ); }); +/** Whether a later call's rules reproduce its observed fields and its omitted members agree with the learned defaults. */ +const agreesLater = ( + context: EffectContext, + draft: WriteDraft, + created: Pick, + call: { readonly body: JsonObject; readonly observed: JsonObject }, +): boolean => + followsCreateRules(context, draft, { ...created, observed: call.observed }) && + !contradictsDefault(draft.plan.rules, call, created.fields); + +/** Whether an echoed member the call left out was stored otherwise than its learned default. */ +const contradictsDefault = ( + rules: readonly FieldRule[], + call: { readonly body: JsonObject; readonly observed: JsonObject }, + written: JsonObject, +): boolean => + rules.some( + (rule) => + rule.rule === "echo" && + !Object.hasOwn(call.body, rule.field) && + Object.hasOwn(call.observed, rule.field) && + !same(call.observed[rule.field], written[rule.field]), + ); + /** Whether a later create preserves generation shapes and its derived fields reproduce the captured observation. */ const followsCreateRules = ( context: EffectContext, @@ -332,10 +425,11 @@ const followsCreateRules = ( draft.plan.rules.every((rule) => { const value = created.fields[rule.field]; const exemplar = draft.observed?.[rule.field]; - if (rule.rule === "composed") + if (rule.rule === "composed" || rule.rule === "reference") return ( !Object.hasOwn(created.observed, rule.field) || - value === created.observed[rule.field] + value === created.observed[rule.field] || + (rule.rule === "reference" && same(value, created.observed[rule.field])) ); if (rule.rule === "clock") return ( @@ -578,25 +672,41 @@ export const applyCreate = ( pathValues: pathValuesOf(step.tool.path_template, collection) ?? call.pathValues, }; - const fields = createdFields(draft.plan, draft.observed!, created, (rule) => - recordedGeneratedValue( - rule, - draft.observed![rule.field], - observed.object[rule.field] ?? null, - ), - ); + const fieldsOfCall = () => + createdFields( + draft.plan, + draft.observed!, + created, + (rule) => + recordedGeneratedValue( + rule, + draft.observed![rule.field], + observed.object[rule.field] ?? null, + ), + referencedIn(context), + ); + // A later call that agrees with the plan widens it, as a later update does: to members it alone sent or left out, an + // array it showed sorted, or a clock spelling. One that disagrees leaves the plan as it was. + if (later) + refineCreate(context, draft, step, { + observed, + call, + collection, + envelope: request?.envelope, + }); + const fields = fieldsOfCall(); const id = fields.ok ? scalarText(fields.value[idField]) : undefined; if ( !fields.ok || id === undefined || context.state.entity(type, collection, id) !== undefined || (later && - !followsCreateRules(context, draft, { - type, - collection, - fields: fields.value, - observed: observed.object, - })) + !agreesLater( + context, + draft, + { type, collection, fields: fields.value }, + { body: call.body, observed: observed.object }, + )) ) { refuse(draft, step, "unexplained-change"); return recorded(); diff --git a/src/domain/analysisView/environmentStateWriteCreateCalls.ts b/src/domain/analysisView/environmentStateWriteCreateCalls.ts new file mode 100644 index 0000000000..2146c5b2e9 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCreateCalls.ts @@ -0,0 +1,241 @@ +import type { JsonValue } from "../jsonValue.js"; +import { same, type JsonObject } from "./environmentStateRules.js"; +import { storedKinds } from "./environmentStateWriteApply.js"; +import { fittedClockFormat } from "./environmentStateWriteClockFormats.js"; +import { + alikeRules, + type FieldRule, + type WriteBody, +} from "./environmentStateWriteRules.js"; + +type Default = { readonly member: string; readonly value: JsonValue }; + +/** What earlier calls showed that a merge must respect: members stored unsorted as sent, and the value showing each clock's width. */ +export interface EarlierEvidence { + readonly kept: ReadonlySet; + readonly clockOf: (field: string) => JsonValue | undefined; +} + +/** The plan's rule for a field and a later call's agree: equal, or clocks one spelling covers given what earlier calls showed. */ +const agreeing = ( + held: FieldRule, + later: FieldRule, + values: { + readonly field: string; + readonly later: JsonValue | undefined; + readonly earlier: EarlierEvidence; + }, +): FieldRule | undefined => { + const alike = alikeRules(held, later, values.earlier.kept); + if (alike !== undefined) return alike; + if (held.rule !== "clock" || later.rule !== "clock") return undefined; + const format = fittedClockFormat( + held.format, + values.later, + values.earlier.clockOf(values.field), + ); + return format === undefined ? undefined : { ...held, format }; +}; + +/** Defaults joined, undefined when two calls showed one member left out at different values. */ +const joinedDefaults = ( + held: readonly Default[], + added: readonly Default[], +): Default[] | undefined => { + const joined = new Map(held.map((entry) => [entry.member, entry.value])); + for (const { member, value } of added) { + if (joined.has(member) && !same(joined.get(member), value)) + return undefined; + joined.set(member, value); + } + return [...joined].map(([member, value]) => ({ member, value })); +}; + +/** What two calls show of one field. */ +interface FieldCalls { + readonly before: FieldRule | undefined; + readonly after: FieldRule | undefined; + readonly exemplar: JsonObject; + readonly later: { readonly entity: JsonObject; readonly body: JsonObject }; + readonly sentBefore: ReadonlySet; + readonly required: readonly string[]; + readonly earlier: EarlierEvidence; +} + +/** A reference an earlier later call added: it stays while a call leaves its member out, and agrees with this call's when it shows it. */ +const addedReference = ( + field: string, + added: Extract, + calls: FieldCalls, +): { readonly rule: FieldRule } | undefined => { + const { after, later } = calls; + if (!Object.hasOwn(later.entity, field)) + return Object.hasOwn(later.body, added.member) + ? undefined + : { rule: added }; + const rule = + after === undefined + ? undefined + : agreeing(added, after, { + field, + later: later.entity[field], + earlier: calls.earlier, + }); + return rule === undefined ? undefined : { rule }; +}; + +/** A field one call shows alone, which must be a reference the other call did not send the member of. */ +const shownAlone = ( + inExemplar: boolean, + calls: FieldCalls, +): { readonly rule: FieldRule } | undefined => { + const own = inExemplar ? calls.before : calls.after; + if (own?.rule !== "reference") return undefined; + const sent = inExemplar + ? Object.hasOwn(calls.later.body, own.member) + : calls.sentBefore.has(own.member); + return sent ? undefined : { rule: own }; +}; + +/** One field's rule in the merged plan, and the default it gives a member; undefined when the calls disagree on it. */ +const mergedField = ( + field: string, + calls: FieldCalls, +): { readonly rule?: FieldRule; readonly default?: Default } | undefined => { + const { before, after, exemplar, later } = calls; + const inExemplar = Object.hasOwn(exemplar, field); + if (!inExemplar && before?.rule === "reference") + return addedReference(field, before, calls); + if (!inExemplar || !Object.hasOwn(later.entity, field)) + return shownAlone(inExemplar, calls); + if (before !== undefined && after !== undefined) { + const rule = agreeing(before, after, { + field, + later: later.entity[field], + earlier: calls.earlier, + }); + return rule === undefined ? undefined : { rule }; + } + // The plan echoes a member this call left out: what it stored there is the member's default. + if (before !== undefined) + return before.rule === "echo" && !Object.hasOwn(later.body, field) + ? { + rule: before, + default: { member: field, value: later.entity[field]! }, + } + : undefined; + // This call echoes a member the plan's calls left out: what they stored there is its default. + if (after !== undefined) + return after.rule === "echo" && !calls.required.includes(field) + ? { rule: after, default: { member: field, value: exemplar[field]! } } + : undefined; + return same(exemplar[field], later.entity[field]) ? {} : undefined; +}; + +/** + * A create plan widened by a later call, explained on its own: both calls show the same fields, but for a reference + * whose member a call without it did not send (one an earlier call added stays), every field both rule they rule alike + * as far as earlier calls allow, a field neither rules holds one value, and where only one echoes a member the other + * left out, the member becomes optional with the value the call that left it out stored. Members either call sent are + * accepted, and only those every call sent are required. Undefined when the calls disagree otherwise, so the later call + * stays unexplained. + */ +export const mergedCreatePlan = ( + plan: { readonly rules: readonly FieldRule[]; readonly body: WriteBody }, + exemplar: JsonObject, + later: { + readonly entity: JsonObject; + readonly body: JsonObject; + readonly rules: readonly FieldRule[]; + readonly contract: WriteBody; + }, + earlier: EarlierEvidence = { + kept: new Set(), + clockOf: (field) => exemplar[field], + }, +): { readonly rules: FieldRule[]; readonly body: WriteBody } | undefined => { + const held = new Map(plan.rules.map((rule) => [rule.field, rule])); + const shown = new Map(later.rules.map((rule) => [rule.field, rule])); + const sentBefore = new Set([ + ...plan.body.accepted, + ...plan.body.fixed.map(({ member }) => member), + ...plan.body.required, + ]); + const merged = new Map(); + const defaults: Default[] = []; + for (const field of new Set([ + ...Object.keys(exemplar), + ...Object.keys(later.entity), + ...held.keys(), + ])) { + const outcome = mergedField(field, { + before: held.get(field), + after: shown.get(field), + exemplar, + later, + sentBefore, + required: plan.body.required, + earlier, + }); + if (outcome === undefined) return undefined; + if (outcome.rule !== undefined) merged.set(field, outcome.rule); + if (outcome.default !== undefined) defaults.push(outcome.default); + } + const body = mergedBody(plan.body, later, exemplar); + const joined = + body === undefined + ? undefined + : joinedDefaults(plan.body.defaults ?? [], defaults); + if (body === undefined || joined === undefined) return undefined; + // The plan's rules keep their order; rules the later call adds follow. + return { + rules: [ + ...plan.rules.flatMap((rule) => { + const kept = merged.get(rule.field); + return kept === undefined ? [] : [kept]; + }), + ...[...merged.values()].filter((rule) => !held.has(rule.field)), + ], + body: joined.length === 0 ? body : { ...body, defaults: joined }, + }; +}; + +/** Two calls' body contracts joined, undefined when a member one accepts with any value the other fixes, or both fix it at different values. */ +const mergedBody = ( + body: WriteBody, + later: { + readonly body: JsonObject; + readonly contract: WriteBody; + readonly entity: JsonObject; + }, + exemplar: JsonObject, +): WriteBody | undefined => { + const fixed = new Map(body.fixed.map((entry) => [entry.member, entry.sent])); + const { contract } = later; + if ( + contract.accepted.some((member) => fixed.has(member)) || + contract.fixed.some( + (entry) => + body.accepted.includes(entry.member) || + (fixed.has(entry.member) && !same(fixed.get(entry.member), entry.sent)), + ) || + contract.envelope !== body.envelope + ) + return undefined; + const added = contract.accepted.filter( + (member) => !body.accepted.includes(member), + ); + return { + ...body, + accepted: [...body.accepted, ...added], + fixed: [ + ...body.fixed, + ...contract.fixed.filter((entry) => !fixed.has(entry.member)), + ], + required: body.required.filter((member) => + Object.hasOwn(later.body, member), + ), + identity: body.identity ?? contract.identity, + kinds: [...body.kinds, ...storedKinds(added, exemplar, later.entity)], + }; +}; diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index 088f9d4dfb..d8f055aa43 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -58,6 +58,8 @@ import { observationNeeds, } from "./environmentStateWriteAddress.js"; import { + alikeRules, + keptOrders, pathValuesOf, siblingTrimmedClocks, type FieldRule, @@ -90,6 +92,8 @@ export interface WriteDraft { readonly steady: Set; /** Unsent clocks a call left as they were in its own second, which it cannot show it stamps. */ readonly unstamped: Set; + /** Members a call stored as sent with strings out of code-point order, which no plan may store sorted. */ + readonly kept: Set; /** A value a held entity or an accepted call showed each clock field at a fixed width in, which keeps it untrimmed. */ readonly fixedWidth: Map; } @@ -260,6 +264,14 @@ export const callOf = ( body, }); +/** The fields of each entity of a type with an id, as the replayed state holds them. */ +export const referencedIn = + (context: EffectContext) => + (type: string, id: string): JsonObject[] => + context.state + .ofType(type) + .flatMap((entity) => (entity.id === id ? [entity.fields] : [])); + export const startOf = ( context: EffectContext, step: WriteStep, @@ -373,8 +385,17 @@ const refineUpdate = ( retainFixed(context, body.fixed.slice(plan.body!.fixed.length)); plan.body = body; const ruled = new Set(plan.rules.map((rule) => rule.field)); + const later = new Map( + evidence.explained.rules.map((rule) => [rule.field, rule]), + ); plan.rules = [ - ...plan.rules, + // A later call may be the first to show an echoed array sorted. + ...plan.rules.map((rule) => { + const shown = later.get(rule.field); + return shown === undefined + ? rule + : (alikeRules(rule, shown, draft.kept) ?? rule); + }), ...evidence.explained.rules.filter((rule) => !ruled.has(rule.field)), ]; }; @@ -542,9 +563,13 @@ export const updateEntity = ( : { readFields: after.readFields }), others: otherFields(context.state.ofType(type), { collection, id }), shown: shownBefore(context.viewOf, entity, after.ordinal), + named: (named) => context.state.named(named), action: needs.action, + references: draft.plan.rules.filter((rule) => rule.rule === "reference"), }), ); + for (const member of keptOrders(call.body, after.object)) + draft.kept.add(member); if (explained.unexplained.length > 0 && !unknown) { refuse(draft, step, "unexplained-change"); return recorded(); @@ -577,6 +602,7 @@ export const updateEntity = ( entity.fields[rule.field], after.object[rule.field], ), + referencedIn(context), ); const kept = keptFields(entity.fields, after.object); const disagrees = @@ -702,4 +728,4 @@ const contradictsPlan = ( ): boolean => contradictsEarlierCalls(draft, explained.rules) || contradictsGeneratedClock(before, after, draft.plan.rules, explained) || - conflictsWithPlan(draft.plan, explained); + conflictsWithPlan(draft.plan, explained, draft.kept); diff --git a/src/domain/analysisView/environmentStateWriteExplain.ts b/src/domain/analysisView/environmentStateWriteExplain.ts index 9bcbdbb3b9..8aef269f26 100644 --- a/src/domain/analysisView/environmentStateWriteExplain.ts +++ b/src/domain/analysisView/environmentStateWriteExplain.ts @@ -26,9 +26,11 @@ import { readClock, sequenceNumber, slugOf, + sortedStrings, type ClockFormat, } from "./environmentStateWriteValues.js"; import { createdComposer } from "./environmentStateWriteComposed.js"; +import { referenceRule } from "./environmentStateWriteReferences.js"; import { fittedClockFormat, showsFixedWidth, @@ -151,6 +153,8 @@ const requestRule = ( const { body, pathValues } = evidence.call; if (Object.hasOwn(body, field) && same(body[field], value)) return { field, rule: "echo" }; + if (Object.hasOwn(body, field) && same(sortedStrings(body[field]), value)) + return { field, rule: "echo", sorted: true }; if (typeof value === "string" && value.length >= COPY_MIN_LENGTH) { const from = index.copies.get(value); if (from !== undefined) return { field, rule: "copy", from }; @@ -257,6 +261,16 @@ const generatedRule = ( return createdNestedClock(field, value, evidence); }; +/** Pinned members less those a reference reads: it rebuilds the object they name for any held entity. */ +const unreferenced = ( + rules: readonly FieldRule[], + pinned: Set, +): Set => { + for (const rule of rules) + if (rule.rule === "reference") pinned.delete(rule.member); + return pinned; +}; + /** Body members the application stored as sent, and the others with what they were sent. */ const bodyOf = ( body: JsonObject, @@ -286,7 +300,10 @@ export const explainCreate = ( for (const [field, value] of Object.entries(evidence.entity)) { const rule = requestRule(evidence, index, field, value) ?? - generatedRule(evidence, field, value); + generatedRule(evidence, field, value) ?? + (evidence.named === undefined + ? undefined + : referenceRule(field, value, evidence.call.body, evidence.named)); if (rule === undefined) unexplained.push(field); else ruled.push(rule); } @@ -337,10 +354,9 @@ export const explainCreate = ( ), ...instantMembers(evidence.call.body, evidence.entity, constants), ]); - const pinned = pinnedEchoes( - evidence.entity, - evidence.call.body, - evidence.type, + const pinned = unreferenced( + rules, + pinnedEchoes(evidence.entity, evidence.call.body, evidence.type), ); const stored = storedMembers( rules.filter((rule) => !stranded.has(rule.field)), @@ -379,6 +395,24 @@ const echoedSlugs = ( return slugs; }; +/** A reference an earlier call showed for a field this call left as it was, when the call names the entity the field repeats through its member. */ +const keptReference = ( + evidence: UpdateEvidence, + field: string, +): FieldRule | undefined => { + const known = evidence.references?.find((rule) => rule.field === field); + if (known === undefined || evidence.named === undefined) return undefined; + const shown = referenceRule( + field, + evidence.after[field], + evidence.call.body, + evidence.named, + ); + return shown?.member === known.member && shown.type === known.type + ? known + : undefined; +}; + /** Explain an update: echoed members, fixed members whose stored value is constant, and the changes it made unsent. */ export const explainUpdate = ( evidence: UpdateEvidence, @@ -428,18 +462,25 @@ export const explainUpdate = ( if ( Object.hasOwn(body, field) || (!Object.hasOwn(evidence.before, field) && - !(evidence.shown ?? []).includes(field)) || - same(evidence.before[field], evidence.after[field]) + !(evidence.shown ?? []).includes(field)) ) continue; + if (same(evidence.before[field], evidence.after[field])) { + const kept = keptReference(evidence, field); + if (kept !== undefined) rules.push(kept); + continue; + } const rule = changeRule(evidence, field, slugs); if (rule === undefined) unexplained.push(field); else rules.push(rule); } rules.push(...pairedStamps(evidence, rules)); + // A member a reference reads names an entity, which any held one may be. + for (const rule of rules) + if (rule.rule === "reference") echoed.add(rule.member); // Serving renews one UUID alike in every field, so the call must have. unexplained.push(...unexplainedRenewals(rules, evidence)); - const pinned = pinnedEchoes(evidence.after, body); + const pinned = unreferenced(rules, pinnedEchoes(evidence.after, body)); const shaped = bodyOf( Object.fromEntries( Object.entries(body).filter(([member]) => member !== identity), diff --git a/src/domain/analysisView/environmentStateWriteReferences.test.ts b/src/domain/analysisView/environmentStateWriteReferences.test.ts new file mode 100644 index 0000000000..9c4d22fb22 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteReferences.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; + +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + FILED_ROWS, + REFILED_ROWS, + TAGGED_ROWS, +} from "../../../tests/fixtures/environmentStateDerivedValues.js"; +import { environmentStateViewOf } from "../../../tests/fixtures/environmentStateWrites.js"; +import type { StateEntity } from "./environmentStateRules.js"; +import { ClockSpellings } from "./environmentStateViewSpellings.js"; +import { alikeRules, type FieldRule } from "./environmentStateWriteRules.js"; +import { + referenceRule, + referencedValue, +} from "./environmentStateWriteReferences.js"; + +const SHELF = { id: 2, name: "Map Room", floor: "upper" }; +const held = ( + type: string, + id: string, + fields: StateEntity["fields"], +): StateEntity => ({ + type, + collection: type, + id, + fields, + view_fields: [], + record_ordinals: [], +}); + +describe("objects that repeat the entity a member names", () => { + it("serves creates that embed the named shelf, and lists show it with the keys they list", () => { + const view = environmentStateViewOf(timedCaptureEvidence(FILED_ROWS)); + expect( + view.writes.find((plan) => plan.tool === "post_books"), + ).toMatchObject({ + served: true, + body: { accepted: expect.arrayContaining(["name", "shelf_id"]) }, + rules: expect.arrayContaining([ + { + field: "shelf", + rule: "reference", + member: "shelf_id", + type: "/api/shelves", + fields: ["id", "name", "floor"], + }, + ]), + }); + expect( + view.views.find((candidate) => candidate.tool === "get_books"), + ).toMatchObject({ + nested_fields: { shelf: ["id", "name"] }, + }); + }); + + it("learns the keys a view shows of a shelf in whatever order each object gives them, keeping the first order", () => { + const spellings = new ClockSpellings(() => undefined); + spellings.note({ shelf: SHELF }, { shelf: { id: 2, name: "Map Room" } }); + spellings.note({ shelf: SHELF }, { shelf: { name: "Map Room", id: 2 } }); + expect(spellings.view).toEqual({ + nested_fields: { shelf: ["id", "name"] }, + }); + spellings.note({ shelf: SHELF }, { shelf: { id: 2 } }); + expect(spellings.view).toEqual({}); + }); + + it("serves updates that keep a shelf a later call names again", () => { + const view = environmentStateViewOf(timedCaptureEvidence(REFILED_ROWS)); + expect(view.writes.find((plan) => plan.effect === "update")).toMatchObject({ + served: true, + body: { accepted: expect.arrayContaining(["name", "shelf_id"]) }, + rules: expect.arrayContaining([ + expect.objectContaining({ field: "shelf", rule: "reference" }), + ]), + }); + }); + + it("names the member whose entity the object repeats, the one named like the field when several do", () => { + const named = (id: string) => + id === "2" + ? [held("/api/shelves", "2", SHELF), held("/api/rooms", "2", SHELF)] + : []; + expect( + referenceRule("shelf", SHELF, { shelf_id: 2 }, named), + ).toBeUndefined(); + const shelves = (id: string) => + id === "2" ? [held("/api/shelves", "2", SHELF)] : []; + expect( + referenceRule("shelf", SHELF, { shelf_id: 2, room_id: 2 }, shelves), + ).toMatchObject({ member: "shelf_id", type: "/api/shelves" }); + expect( + referenceRule("shelf", { id: 2 }, { shelf_id: 2 }, shelves), + ).toBeUndefined(); + // One id naming a shelf in two collections names neither. + expect( + referenceRule("shelf", SHELF, { shelf_id: 2 }, (id) => + id === "2" + ? [ + held("/api/shelves", "2", SHELF), + { ...held("/api/shelves", "2", SHELF), collection: "/api/annex" }, + ] + : [], + ), + ).toBeUndefined(); + expect( + referenceRule( + "shelf", + { ...SHELF, floor: "lower" }, + { shelf_id: 2 }, + shelves, + ), + ).toBeUndefined(); + }); + + it("refuses a call naming an entity the environment does not hold, or holds in several collections", () => { + const rule: Extract = { + field: "shelf", + rule: "reference", + member: "shelf_id", + type: "/api/shelves", + fields: ["id", "name"], + }; + expect(referencedValue(rule, { shelf_id: 9 }, () => [])).toMatchObject({ + ok: false, + }); + expect(referencedValue(rule, {}, () => [])).toEqual({ + ok: true, + value: undefined, + }); + expect(referencedValue(rule, { shelf_id: 2 }, () => [SHELF])).toEqual({ + ok: true, + value: { id: 2, name: "Map Room" }, + }); + expect( + referencedValue(rule, { shelf_id: 2 }, () => [SHELF, SHELF]), + ).toMatchObject({ + ok: false, + error: expect.stringContaining("2 collections"), + }); + }); +}); + +describe("arrays an application stores sorted", () => { + it("does not sort tags an earlier create stored out of order as sent", () => { + const kept = TAGGED_ROWS.map((row, index): TimedCaptureRow => { + if (index !== 2 && index !== 3) return row; + const [method, path, response, ...rest] = row; + return [ + method, + path, + { ...(response as object), tags: ["zoning", "maps"] }, + ...rest, + ]; + }); + const plan = environmentStateViewOf(timedCaptureEvidence(kept)).writes.find( + (candidate) => candidate.tool === "post_books", + ); + expect(plan?.rules).toContainEqual({ field: "tags", rule: "echo" }); + expect(plan).toMatchObject({ served: false }); + }); + + it("keeps an echo sorted only while no call kept its strings out of order", () => { + const plain: FieldRule = { field: "tags", rule: "echo" }; + const sorted: FieldRule = { field: "tags", rule: "echo", sorted: true }; + expect(alikeRules(plain, sorted)).toEqual(sorted); + expect(alikeRules(sorted, plain)).toEqual(sorted); + expect(alikeRules(plain, sorted, new Set(["tags"]))).toBeUndefined(); + expect(alikeRules(sorted, plain, new Set(["tags"]))).toBeUndefined(); + }); + + it("serves creates that store sent tags in code-point order", () => { + const view = environmentStateViewOf(timedCaptureEvidence(TAGGED_ROWS)); + expect( + view.writes.find((plan) => plan.tool === "post_books"), + ).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "tags", rule: "echo", sorted: true }, + ]), + }); + }); +}); diff --git a/src/domain/analysisView/environmentStateWriteReferences.ts b/src/domain/analysisView/environmentStateWriteReferences.ts new file mode 100644 index 0000000000..5d5a68525a --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteReferences.ts @@ -0,0 +1,99 @@ +import type { JsonValue } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import { + isObject, + same, + scalarText, + type JsonObject, + type StateEntity, +} from "./environmentStateRules.js"; +import type { FieldRule } from "./environmentStateWriteRules.js"; + +type ReferenceRule = Extract; + +/** The fields of each entity of a type with an id the state holds, one per collection holding one. */ +export type ReferencedEntity = ( + type: string, + id: string, +) => readonly JsonObject[]; + +/** A member name with its trailing id dropped and lower case, to tie `parentId` to `parent`. */ +const stemOf = (name: string): string => + name + .replace(/(?:Id|ID|_id|_ID|Ids|_ids)$/u, "") + .toLowerCase() + .replace(/[^a-z0-9]/gu, ""); + +/** Whether every member of an object holds the value an entity holds there. */ +const projects = (object: JsonObject, fields: JsonObject): boolean => + Object.entries(object).every( + ([key, value]) => Object.hasOwn(fields, key) && same(fields[key], value), + ); + +/** + * A created or updated object field that repeats fields of the entity a sent member names, as the state holds it: a + * parent beside `parentId`. Several members naming such entities resolve to the one named like the field; undefined + * when none does, or when that leaves more than one. An id that names entities of one type in several collections + * names none, since serving could not tell which. + */ +export const referenceRule = ( + field: string, + value: JsonValue | undefined, + body: JsonObject, + named: (id: string) => readonly StateEntity[], +): ReferenceRule | undefined => { + if (!isObject(value) || Object.keys(value).length < 2) return undefined; + const matches = new Map(); + for (const [member, sent] of Object.entries(body)) { + const id = scalarText(sent); + if (id === undefined || id.length === 0) continue; + const held = named(id); + for (const entity of held) + if ( + projects(value, entity.fields) && + held.filter((other) => other.type === entity.type).length === 1 + ) + matches.set(`${member} ${entity.type}`, { member, type: entity.type }); + } + const candidates = [...matches.values()]; + const chosen = + candidates.length === 1 + ? candidates + : candidates.filter(({ member }) => stemOf(member) === stemOf(field)); + return chosen.length === 1 + ? { + field, + rule: "reference", + member: chosen[0]!.member, + type: chosen[0]!.type, + fields: Object.keys(value), + } + : undefined; +}; + +/** A reference's value for a call: the named entity's fields it lists, or undefined when the call does not send its member. */ +export const referencedValue = ( + rule: ReferenceRule, + body: JsonObject, + referenced: ReferencedEntity | undefined, +): Result => { + if (!Object.hasOwn(body, rule.member)) return ok(undefined); + const id = scalarText(body[rule.member]); + const held = id === undefined ? [] : (referenced?.(rule.type, id) ?? []); + const [fields] = held; + if (fields === undefined) + return err( + `Member ${rule.member} names no entity of ${rule.type} the environment holds, which ${rule.field} repeats.`, + ); + if (held.length > 1) + return err( + `Member ${rule.member} names entities of ${rule.type} in ${held.length} collections, so ${rule.field} cannot tell which it repeats.`, + ); + return ok( + Object.fromEntries( + rule.fields.flatMap((key) => + Object.hasOwn(fields, key) ? [[key, fields[key]!]] : [], + ), + ), + ); +}; diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts index badc28748f..843b96ab01 100644 --- a/src/domain/analysisView/environmentStateWriteRules.ts +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -4,10 +4,14 @@ import { isParam, normalizedPath, pathSegmentMatches, + same, segmentsOf, type JsonObject, } from "./environmentStateRules.js"; -import type { ClockFormat } from "./environmentStateWriteValues.js"; +import { + sortedStrings, + type ClockFormat, +} from "./environmentStateWriteValues.js"; import { showsFixedWidth, trimmedLike, @@ -70,7 +74,8 @@ export type GeneratedRule = /** How one field of a written entity gets its value; fields no rule names keep the observed value. */ export type FieldRule = | GeneratedRule - | { readonly field: string; readonly rule: "echo" } + /** The sent member stored as sent, or with its strings in code-point order when `sorted`. */ + | { readonly field: string; readonly rule: "echo"; readonly sorted?: true } | { readonly field: string; readonly rule: "copy"; readonly from: string } | { readonly field: string; readonly rule: "path"; readonly from: string } | { readonly field: string; readonly rule: "slug"; readonly from: string } @@ -81,6 +86,14 @@ export type FieldRule = readonly rule: "composed"; readonly parts: ComposedPart[]; } + /** An object holding `fields` of the entity of `type` the sent `member` names, as the state holds it; absent when the call does not send it. */ + | { + readonly field: string; + readonly rule: "reference"; + readonly member: string; + readonly type: string; + readonly fields: string[]; + } | { readonly field: string; readonly rule: "increment" } /** An update stores `value` whenever the call sends the member, as the call that showed it did. */ | { @@ -143,6 +156,31 @@ export const siblingTrimmedClocks = ( }); }; +/** + * Two calls' rules for a field as one plan holds them: equal, or echoes one of which sorted its strings (a call whose + * array was already in order cannot show it), kept sorted unless a call kept strings out of order. Undefined when they + * differ otherwise. + */ +export const alikeRules = ( + held: FieldRule, + later: FieldRule, + kept: ReadonlySet = new Set(), +): FieldRule | undefined => { + if (held.rule !== "echo" || later.rule !== "echo") + return same(held, later) ? held : undefined; + if ((held.sorted === true) === (later.sorted === true)) return held; + if (kept.has(held.field)) return undefined; + return held.sorted === true ? held : later; +}; + +/** Members a call sent as strings out of code-point order and stored as sent, which shows the application keeps their order. */ +export const keptOrders = (body: JsonObject, stored: JsonObject): string[] => + Object.keys(body).filter( + (member) => + sortedStrings(body[member]) !== undefined && + same(stored[member], body[member]), + ); + /** One part of a composed string: literal text, or another written field as written, slugged, or as a clock's calendar digits. */ export type ComposedPart = | { readonly text: string } @@ -176,6 +214,10 @@ export interface WriteBody { readonly kinds: { readonly member: string; readonly kind: JsonKind }[]; /** The member a call wraps its object in, as a one-element array. */ readonly envelope?: string | undefined; + /** The value a create stores for an accepted member a call leaves out, as a recorded call that left it out showed. */ + readonly defaults?: + | { readonly member: string; readonly value: JsonValue }[] + | undefined; } /** diff --git a/src/domain/analysisView/environmentStateWriteValues.ts b/src/domain/analysisView/environmentStateWriteValues.ts index 8ab0d19bcb..79d1fe7030 100644 --- a/src/domain/analysisView/environmentStateWriteValues.ts +++ b/src/domain/analysisView/environmentStateWriteValues.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import type { JsonValue } from "../jsonValue.js"; import { readRecordedDateTime } from "../offsetDateTime.js"; +import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; /** How a recorded clock value is spelled: ISO-8601 with its separator, fraction digits and zone, or an epoch number. */ export type ClockFormat = @@ -194,6 +195,28 @@ export const generatedIntegerDigits = ( : undefined; }; +/** A sent array of strings as an application stores it sorted: in code-point order, when that differs from the sent order and shows two distinct strings. */ +export const sortedStrings = ( + sent: JsonValue | undefined, +): string[] | undefined => { + if ( + !Array.isArray(sent) || + !sent.every((item) => typeof item === "string") || + new Set(sent).size < 2 + ) + return undefined; + const sorted = (sent as string[]).toSorted(compareUnicodeCodePoints); + return sorted.some((item, index) => item !== sent[index]) + ? sorted + : undefined; +}; + +/** The value an echo rule stores for a sent value: as sent, or its strings sorted. */ +export const echoed = ( + rule: { readonly sorted?: true }, + sent: JsonValue, +): JsonValue => (rule.sorted === true ? (sortedStrings(sent) ?? sent) : sent); + /** Whether a generated integer retains the exemplar's digit count and JSON kind. */ export const likeGeneratedInteger = ( value: JsonValue | undefined, diff --git a/tests/boundary/mcp/environmentServeWritesMcp.test.ts b/tests/boundary/mcp/environmentServeWritesMcp.test.ts index ef053d67d2..228678bdd2 100644 --- a/tests/boundary/mcp/environmentServeWritesMcp.test.ts +++ b/tests/boundary/mcp/environmentServeWritesMcp.test.ts @@ -23,6 +23,7 @@ import { SECOND_BOOK, renamedBook, } from "../../fixtures/environmentStateWrites.js"; +import { referencedCreateRows } from "../../fixtures/environmentStateDerivedValues.js"; import { unsupportedInputLiteralSchemaError } from "../../fixtures/mcpClientSchemaConstraints.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { startListeningProcess } from "../../support/mcp/streamableHttpProcess.js"; @@ -409,3 +410,40 @@ processTest( }, HTTP_BOUNDARY_TIMEOUT_MS, ); + +processTest( + "withholds a create from public tools when its sent reference contradicts the recorded response", + async ({ processes }) => { + const root = await createTestTempDirectory("rea-create-reference-mcp-"); + const rows = referencedCreateRows({ + id: 1, + name: "Different Hall", + floor: "ground", + }); + const files = await writeEnvironmentServeFiles( + root, + environmentStateServeFixture(timedCaptureEvidence(rows)), + ); + const started = await startListeningProcess(processes, [ + "environment-serve", + files.manifestPath, + "--evidence", + files.capturePath, + "--port", + "0", + ]); + const client = new Client({ + name: "contradictory-reference", + version: "1", + }); + await client.connect(new StreamableHTTPClientTransport(started.url)); + try { + const { tools } = await client.listTools(); + expect(tools.map(({ name }) => name)).toContain("get_books"); + expect(tools.map(({ name }) => name)).not.toContain("post_books"); + } finally { + await client.close(); + } + }, + HTTP_BOUNDARY_TIMEOUT_MS, +); diff --git a/tests/fixtures/environmentStateDerivedValues.ts b/tests/fixtures/environmentStateDerivedValues.ts index dd18c8fab8..76a2f2f006 100644 --- a/tests/fixtures/environmentStateDerivedValues.ts +++ b/tests/fixtures/environmentStateDerivedValues.ts @@ -53,6 +53,21 @@ export const TAG_ROWS: readonly TimedCaptureRow[] = [ ["GET", "/api/tags", [FIRST_TAG, SECOND_TAG, HEAVY_TAG, BORROWED_TAG]], ]; +/** A held tag whose update clock keeps a trailing zero, so the application spells that clock at a fixed width. */ +const FIXED_SECOND_TAG = { + ...SECOND_TAG, + updatedAt: "2026-10-01T10:00:00.223456790Z", +}; + +/** The tag creates beside a held tag that shows the update clock at a fixed width, which the later create's trimmed clock contradicts. */ +export const HELD_FIXED_TAG_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/tags", [FIRST_TAG, FIXED_SECOND_TAG]], + ["GET", `/api/tags/${numberedUuid(1)}`, FIRST_TAG], + ["GET", `/api/tags/${numberedUuid(2)}`, FIXED_SECOND_TAG], + ...TAG_ROWS.slice(3, 7), + ["GET", "/api/tags", [FIRST_TAG, FIXED_SECOND_TAG, HEAVY_TAG, BORROWED_TAG]], +]; + /** A later tag create whose update clock Go trims while its creation clock keeps a trailing zero. */ const FIXED_CREATED_TAG = tag( 4, @@ -320,6 +335,215 @@ export const TAG_UPDATE_ROWS: readonly TimedCaptureRow[] = [ ["GET", `/api/tags/${numberedUuid(2)}`, RENAMED_SECOND_TAG], ]; +/** A book shelved as the application shelves it, on `general` when the call names no shelf. */ +const shelved = ( + id: number, + name: string, + shelf: string, + at: string, +): Entity => ({ + id, + name, + shelf, + created_at: at, +}); + +const SHELVED_SEEDS = [ + shelved(1, "First Book", "general", "2026-10-01T09:00:00Z"), + shelved(2, "Second Book", "general", "2026-10-01T10:00:00Z"), +]; + +/** Two book creates, one naming a shelf and one leaving it out, in the order given, each read back. */ +export const shelvedRows = ( + first: { readonly name: string; readonly shelf?: string }, + second: { readonly name: string; readonly shelf?: string }, + stored: (sent: string) => string = (sent) => sent, +): TimedCaptureRow[] => { + const third = shelved( + 3, + first.name, + first.shelf === undefined ? "general" : stored(first.shelf), + bookClock(3), + ); + const fourth = shelved( + 4, + second.name, + second.shelf === undefined ? "general" : stored(second.shelf), + bookClock(5), + ); + return [ + ["GET", "/api/books", SHELVED_SEEDS], + ["GET", "/api/books/1", SHELVED_SEEDS[0]!], + ["GET", "/api/books/2", SHELVED_SEEDS[1]!], + ["POST", "/api/books", third, 201, first], + ["GET", "/api/books/3", third], + ["POST", "/api/books", fourth, 201, second], + ["GET", "/api/books/4", fourth], + ["GET", "/api/books", [...SHELVED_SEEDS, third, fourth]], + ]; +}; + +/** The shelved creates, then one that leaves the shelf out yet stores another shelf than the default, shown only in its answer. */ +export const misshelvedRows = (): TimedCaptureRow[] => { + const rows = shelvedRows( + { name: "Third Book" }, + { name: "Fourth Book", shelf: "rare" }, + ); + const fifth = shelved(5, "Fifth Book", "attic", bookClock(rows.length)); + return [...rows, ["POST", "/api/books", fifth, 201, { name: "Fifth Book" }]]; +}; + +const SHELVES = [ + { id: 1, name: "Main Hall", floor: "ground" }, + { id: 2, name: "Map Room", floor: "upper" }, +]; + +/** A book as its item read shows it: its shelf whole, which its list read shows by id and name only. */ +const filed = ( + id: number, + name: string, + shelf: number, + at: string, +): Entity => ({ + id, + name, + shelf: SHELVES[shelf - 1]!, + created_at: at, +}); + +const listed = (book: Entity): Entity => { + const { id, name } = book["shelf"] as { id: number; name: string }; + return { ...book, shelf: { id, name } }; +}; + +const FILED_SEED = filed(1, "Tide Tables", 1, "2026-10-01T09:00:00Z"); +const ATLAS = filed(2, "Coastal Atlas", 2, bookClock(5)); +const CHARTS = filed(3, "Harbour Charts", 1, bookClock(8)); + +/** Two book creates naming a shelf by `shelf_id`, whose answers and item reads show the shelf whole and whose list shows it by id and name. */ +export const FILED_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/shelves", SHELVES], + ["GET", "/api/shelves/1", SHELVES[0]!], + ["GET", "/api/shelves/2", SHELVES[1]!], + ["GET", "/api/books", [listed(FILED_SEED)]], + ["GET", "/api/books/1", FILED_SEED], + ["POST", "/api/books", ATLAS, 201, { name: "Coastal Atlas", shelf_id: 2 }], + ["GET", "/api/books/2", ATLAS], + ["GET", "/api/books", [listed(FILED_SEED), listed(ATLAS)]], + ["POST", "/api/books", CHARTS, 201, { name: "Harbour Charts", shelf_id: 1 }], + ["GET", "/api/books/3", CHARTS], + ["GET", "/api/books", [listed(FILED_SEED), listed(ATLAS), listed(CHARTS)]], +]; + +/** Two creates whose last answer can contradict the named shelf, followed only by a list omitting that field. */ +export const referencedCreateRows = (shelf: JsonValue): TimedCaptureRow[] => { + const last = { ...CHARTS, shelf }; + const summary = ({ id, name }: Entity): Entity => ({ id: id!, name: name! }); + return [ + ...FILED_ROWS.filter(([, path]) => path.startsWith("/api/shelves")), + ["GET", "/api/books", [summary(FILED_SEED)]], + ["GET", "/api/books/1", FILED_SEED], + ["POST", "/api/books", ATLAS, 201, { name: "Coastal Atlas", shelf_id: 2 }], + ["GET", "/api/books/2", ATLAS], + ["GET", "/api/books", [FILED_SEED, ATLAS].map(summary)], + ["POST", "/api/books", last, 201, { name: "Harbour Charts", shelf_id: 1 }], + ["GET", "/api/books", [FILED_SEED, ATLAS, last].map(summary)], + ]; +}; + +/** A book moved to another shelf by `shelf_id`, then renamed by a call that sends the shelf it already has. */ +export const REFILED_ROWS: readonly TimedCaptureRow[] = (() => { + const moved = filed(1, "Tide Tables", 2, "2026-10-01T09:00:00Z"); + const renamed = { ...moved, name: "Tide Tables, Revised" }; + return [ + ...FILED_ROWS.slice(0, 5), + ["PATCH", "/api/books/1", moved, 200, { shelf_id: 2 }], + ["GET", "/api/books/1", moved], + ["GET", "/api/books", [listed(moved)]], + [ + "PATCH", + "/api/books/1", + renamed, + 200, + { name: "Tide Tables, Revised", shelf_id: 2 }, + ], + ["GET", "/api/books/1", renamed], + ["GET", "/api/books", [listed(renamed)]], + ]; +})(); + +/** Two book creates, one naming a shelf by `shelf_id` and one leaving it out, in the order given, each read back and listed. */ +export const optionallyFiledRows = (shelfFirst: boolean): TimedCaptureRow[] => { + const unfiled = (id: number, at: string): Entity => ({ + id, + name: "Field Notes", + created_at: at, + }); + const [first, second] = shelfFirst + ? [filed(2, "Coastal Atlas", 2, bookClock(5)), unfiled(3, bookClock(8))] + : [unfiled(2, bookClock(5)), filed(3, "Coastal Atlas", 2, bookClock(8))]; + const sent = (book: Entity) => + "shelf" in book + ? { name: book["name"]!, shelf_id: 2 } + : { name: book["name"]! }; + const shown = (book: Entity) => ("shelf" in book ? listed(book) : book); + return [ + ...FILED_ROWS.slice(0, 5), + ["POST", "/api/books", first, 201, sent(first)], + ["GET", "/api/books/2", first], + ["GET", "/api/books", [listed(FILED_SEED), shown(first)]], + ["POST", "/api/books", second, 201, sent(second)], + ["GET", "/api/books/3", second], + ["GET", "/api/books", [listed(FILED_SEED), shown(first), shown(second)]], + ]; +}; + +const tagged = ( + id: number, + name: string, + tags: string[], + at: string, +): Entity => ({ + id, + name, + tags, + created_at: at, +}); + +/** Two book creates sending tags out of order, which the application stores sorted. */ +export const TAGGED_ROWS: readonly TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [tagged(1, "Tide Tables", ["coast", "tides"], "2026-10-01T09:00:00Z")], + ], + [ + "GET", + "/api/books/1", + tagged(1, "Tide Tables", ["coast", "tides"], "2026-10-01T09:00:00Z"), + ], + [ + "POST", + "/api/books", + tagged(2, "Atlas", ["maps", "zoning"], bookClock(2)), + 201, + { name: "Atlas", tags: ["zoning", "maps"] }, + ], + ["GET", "/api/books/2", tagged(2, "Atlas", ["maps", "zoning"], bookClock(2))], + [ + "POST", + "/api/books", + tagged(3, "Charts", ["harbour", "sea"], bookClock(4)), + 201, + { name: "Charts", tags: ["sea", "harbour"] }, + ], + [ + "GET", + "/api/books/3", + tagged(3, "Charts", ["harbour", "sea"], bookClock(4)), + ], +]; + /** The same renames with an archive the gate does not model before the second, which leaves that tag unknown. */ export const UNKNOWN_TAG_UPDATE_ROWS: readonly TimedCaptureRow[] = [ ...TAG_UPDATE_ROWS.slice(0, 5), From 6bcd243ca486b6b16f119c7693c605b48d576882 Mon Sep 17 00:00:00 2001 From: Syed Rishad Date: Sat, 24 Oct 2026 18:34:11 -0400 Subject: [PATCH 22/37] fix(verifiers): expect the session-continuation regex overflow reason (#7045) overflow into the remediation and ended the reason with the session continuation sentence, but the real-Ghidra boundary check still required the old reason text, so every lane that runs verifyGhidraBoundaries failed on real Ghidra. --- scripts/lib/real-ghidra-boundaries.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index d2c7607613..67078d31b8 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -529,9 +529,11 @@ export async function verifyGhidraBoundaries( regexError.remediation.action, /literal mode or simplify the regex/u, ); + // The retry advice lives in the remediation; the reason states that the + // session survived the overflow. assert.match( regexError.details.reason, - /exhausted its stack.*Use literal mode/u, + /exhausted its stack.*session and annotations remain available/u, ); rejectedCalls++; const nestedPattern = "(".repeat(12000) + "a" + ")".repeat(12000); From 1ff275fbf752b117575b5726dc86a1ff82cc011e Mon Sep 17 00:00:00 2001 From: Dery Ferdika <114888155+DeryFerd@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:34:26 +0700 Subject: [PATCH 23/37] fix(browser): admit passive capture events against the shared retained budget (#7043) Passive page capture retained console entries and websocket frames with no aggregate ceiling. Every sibling retention surface already enforces WEB_RUNTIME_LIMITS.retainedEventBytes (8 MiB): the observation session's pre-arm buffer and document-request identities, the browser runtime sources, requests, and metadata, the V8 inspector's retained metadata, and this same capture's script metadata, which has its own count-and-bytes budget. A page that logs a few hundred megabytes of console text, or sends text frames whose JSON shapes carry hundreds of thousands of keys, grew the capture without limit until the process ran out of memory; the MCP response budget runs after the handler, so the growth happened before any typed refusal could exist. Admit each console entry and websocket frame against the same shared budget before retaining it. A refused entry is counted through the completeness tracker's dropped-events vocabulary, which already carries console_events and websocket_frames kinds and was so far used only for late scripts, so the inspection reports a truncated inventory with the dropped count instead of silently overflowing. Console entry bytes measure the retained representation: the entry's type, url, argument-type strings, and the text capture's retained bytes, which the handler already computed per entry. A websocket frame retains its shape summary rather than the payload, and the shape grows with the payload's own keys, so the frame is charged the serialized size of the shape it retains. beginAuthorizedFrame resets the counter together with the rest of the per-frame state. Out of scope: response metadata and agent hints, whose rows are fixed-shape facts already bounded per response, and any new setting. The shared 8 MiB constant is reused as-is. Validation: RED then green in src/browser/CdpCaptureEvents.eventBudget.test.ts, where the console case previously retained every oversized entry and the websocket case retained the over-budget frame; 26 tests across the neighboring console, websocket, coverage, and completeness suites pass. On this Windows host the wider browser project has 26 pre-existing failures in three Electron/WebBundle worker-spawn files, reproduced identically with this change stashed (26 failed, 644 passed on the pristine base; 26 failed, 647 passed with it on the same file set). tsc --noEmit over the changed set exits 0, oxlint reports 0 errors, oxfmt --check is clean, and verify:module-boundaries passes. No real browser session was driven on this host; the browser CI lane owns that coverage. Each retained entry now carries a conservative fixed overhead, and each captured console text value a wrapper cost, so a stream of minimal entries cannot admit millions of objects into the budget when shapes are disabled; a mixed-kind test locks that console entries and websocket frames spend one shared allowance. --- src/browser/CdpCaptureEventHandlers.ts | 55 ++++++- src/browser/CdpCaptureEventState.ts | 2 + .../CdpCaptureEvents.eventBudget.test.ts | 146 ++++++++++++++++++ src/browser/CdpCaptureEvents.ts | 2 + 4 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 src/browser/CdpCaptureEvents.eventBudget.test.ts diff --git a/src/browser/CdpCaptureEventHandlers.ts b/src/browser/CdpCaptureEventHandlers.ts index d11652b188..a3d6bd7b84 100644 --- a/src/browser/CdpCaptureEventHandlers.ts +++ b/src/browser/CdpCaptureEventHandlers.ts @@ -2,6 +2,7 @@ import type { WebPageInspection } from "../domain/browserObservationSchemas.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; import { inferJsonShape } from "../domain/jsonShape.js"; import { paddedBase64ByteLength } from "../domain/paddedBase64.js"; +import { WEB_RUNTIME_LIMITS } from "../domain/webRuntime.js"; import { safeResponseMetadata } from "./CdpSafeMetadata.js"; import { allowedSanitizedUrl, @@ -151,6 +152,13 @@ const scriptMetadataStringBytes = (script: { const stringByteLength = (value: string | null): number => value === null ? 0 : Buffer.byteLength(value, "utf8"); +/** Approximate fixed bytes of one retained console entry or frame object, so a + * stream of minimal entries cannot admit millions of objects into the budget. */ +const RETAINED_EVENT_ENTRY_OVERHEAD_BYTES = 64; +/** Approximate fixed bytes of one captured console text value wrapper beyond + * its retained text. */ +const RETAINED_EVENT_TEXT_VALUE_BYTES = 32; + const sourceMapForScript = ( value: unknown, scriptUrl: string, @@ -473,15 +481,38 @@ export const handleConsoleAPICalled = ( argumentTypes.includes(undefined) ) state.completeness.exclude("console_events", "invalid_protocol_value"); + const type = cdpStringValue(params.type) ?? "unknown"; + const textCapture = captureConsoleText(state, arguments_); + // The shared retained-event budget bounds the whole console inventory, so a + // page cannot grow the capture without limit; a refused entry stays counted + // as a dropped event instead of vanishing. + const entryBytes = + RETAINED_EVENT_ENTRY_OVERHEAD_BYTES + + stringByteLength(type) + + stringByteLength(source.url) + + (argumentTypes ?? []).reduce( + (sum, argumentType) => sum + stringByteLength(argumentType ?? "unknown"), + 0, + ) + + textCapture.retained_bytes + + textCapture.values.length * RETAINED_EVENT_TEXT_VALUE_BYTES; + if ( + entryBytes > + WEB_RUNTIME_LIMITS.retainedEventBytes - state.retainedEventBytes + ) { + state.completeness.drop("console_events"); + return; + } + state.retainedEventBytes += entryBytes; state.console.push({ - type: cdpStringValue(params.type) ?? "unknown", + type, timestamp: timestamp ?? null, argument_types: argumentTypes?.map((argumentType) => argumentType ?? "unknown") ?? null, url: source.url, line: integerOrNull(frame.lineNumber), column: integerOrNull(frame.columnNumber), - text_capture: captureConsoleText(state, arguments_), + text_capture: textCapture, }); }; @@ -551,11 +582,29 @@ export const handleWebSocketFrame = ( : (paddedBase64ByteLength(payload) ?? null); if (opcode === null || payloadBytes === null) state.completeness.exclude("websocket_frames", "invalid_protocol_value"); + const payloadShape = captureWebSocketShape(state, payload, opcode); + // The frame retains its shape summary, not the payload, and the shape grows + // with the payload's own keys; the shared retained-event budget bounds the + // retained frames and a refused frame stays counted as a dropped event. + const entryBytes = + RETAINED_EVENT_ENTRY_OVERHEAD_BYTES + + stringByteLength(direction) + + (payloadShape === null + ? 0 + : Buffer.byteLength(JSON.stringify(payloadShape), "utf8")); + if ( + entryBytes > + WEB_RUNTIME_LIMITS.retainedEventBytes - state.retainedEventBytes + ) { + state.completeness.drop("websocket_frames"); + return; + } + state.retainedEventBytes += entryBytes; connection.events.push({ direction, opcode, payload_bytes: payloadBytes, - payload_shape: captureWebSocketShape(state, payload, opcode), + payload_shape: payloadShape, }); }; diff --git a/src/browser/CdpCaptureEventState.ts b/src/browser/CdpCaptureEventState.ts index 778c54812c..331fc2807e 100644 --- a/src/browser/CdpCaptureEventState.ts +++ b/src/browser/CdpCaptureEventState.ts @@ -12,6 +12,8 @@ export interface CdpCaptureEventsState { rejectedSourceMapMetadataBudgetCount: number; readonly scriptMetadataBytesById: Map; retainedScriptMetadataBytes: number; + /** Console-entry and websocket-frame bytes admitted against the shared retained-event budget. */ + retainedEventBytes: number; readonly executionContextFrames: Map; readonly network: Map; readonly networkRequestTimestamps: Map; diff --git a/src/browser/CdpCaptureEvents.eventBudget.test.ts b/src/browser/CdpCaptureEvents.eventBudget.test.ts new file mode 100644 index 0000000000..496aec5537 --- /dev/null +++ b/src/browser/CdpCaptureEvents.eventBudget.test.ts @@ -0,0 +1,146 @@ +import { expect, it } from "vitest"; + +import { inspectWebPageInputSchema } from "../domain/browserObservation.js"; +import { WEB_RUNTIME_LIMITS } from "../domain/webRuntime.js"; +import { CdpCaptureEvents } from "./CdpCaptureEvents.js"; + +const origin = "http://127.0.0.1:43127"; + +const createEvents = (extra: Record = {}): CdpCaptureEvents => + new CdpCaptureEvents( + inspectWebPageInputSchema.parse({ + cdp_endpoint: origin, + allowed_origins: [origin], + target_id: "page", + include_console_text: true, + include_websocket_shapes: true, + ...extra, + }), + new Set([origin]), + ); + +it("admits console events until the shared retained-event budget and reports the dropped count", () => { + const events = createEvents(); + const text = "x".repeat(1024 * 1024); + const total = + Math.ceil(WEB_RUNTIME_LIMITS.retainedEventBytes / text.length) + 2; + for (let index = 0; index < total; index += 1) + events.ingest({ + method: "Runtime.consoleAPICalled", + params: { + type: "log", + timestamp: index, + args: [{ type: "string", value: text }], + stackTrace: { + callFrames: [ + { url: `${origin}/app.js`, lineNumber: 0, columnNumber: 0 }, + ], + }, + }, + }); + expect(events.console.length).toBeLessThan(total); + expect(events.console.length).toBeGreaterThan(0); + const snapshot = events.completeness.snapshot(); + const dropped = total - events.console.length; + expect(snapshot.dropped_events.console_events).toBe(dropped); + expect(snapshot.dropped_events.total).toBe(dropped); +}); + +it("keeps a small console inventory admitted under the budget untouched", () => { + const events = createEvents(); + for (let index = 0; index < 16; index += 1) + events.ingest({ + method: "Runtime.consoleAPICalled", + params: { + type: "log", + timestamp: index, + args: [{ type: "string", value: `entry ${String(index)}` }], + stackTrace: { + callFrames: [ + { url: `${origin}/app.js`, lineNumber: 0, columnNumber: 0 }, + ], + }, + }, + }); + expect(events.console).toHaveLength(16); + expect(events.completeness.snapshot().dropped_events.total).toBe(0); +}); + +it("drops a websocket frame whose retained shape alone exceeds the shared budget", () => { + const events = createEvents(); + events.ingest({ + method: "Network.webSocketCreated", + params: { requestId: "socket", url: `ws://${origin.slice(7)}/live` }, + }); + events.ingest({ + method: "Network.webSocketFrameReceived", + params: { + requestId: "socket", + response: { opcode: 1, payloadData: '{"small":"frame"}' }, + }, + }); + const keys = Array.from( + { length: 900_000 }, + (_value, index) => `key${String(index)}`, + ); + events.ingest({ + method: "Network.webSocketFrameReceived", + params: { + requestId: "socket", + response: { + opcode: 1, + payloadData: `{${keys.map((key) => `"${key}":"v"`).join(",")}}`, + }, + }, + }); + const connection = events.webSocketConnectionsById.get("socket"); + expect(connection?.events).toHaveLength(1); + const snapshot = events.completeness.snapshot(); + expect(snapshot.dropped_events.websocket_frames).toBe(1); + expect(snapshot.dropped_events.total).toBe(1); +}); + +it("spends one shared allowance across console entries and websocket frames", () => { + const events = createEvents(); + const text = "x".repeat(1024 * 1024); + // Seven megabyte console entries leave roughly one megabyte of the shared + // allowance, so a shape that fits an empty allowance no longer fits. + for (let index = 0; index < 7; index += 1) + events.ingest({ + method: "Runtime.consoleAPICalled", + params: { + type: "log", + timestamp: index, + args: [{ type: "string", value: text }], + stackTrace: { + callFrames: [ + { url: `${origin}/app.js`, lineNumber: 0, columnNumber: 0 }, + ], + }, + }, + }); + expect(events.console).toHaveLength(7); + events.ingest({ + method: "Network.webSocketCreated", + params: { requestId: "socket", url: `ws://${origin.slice(7)}/live` }, + }); + const keys = Array.from( + { length: 200_000 }, + (_value, index) => `key${String(index)}`, + ); + events.ingest({ + method: "Network.webSocketFrameReceived", + params: { + requestId: "socket", + response: { + opcode: 1, + payloadData: `{${keys.map((key) => `"${key}":"v"`).join(",")}}`, + }, + }, + }); + expect(events.webSocketConnectionsById.get("socket")?.events).toHaveLength(0); + const snapshot = events.completeness.snapshot(); + expect(snapshot.dropped_events.console_events).toBe(0); + expect(snapshot.dropped_events.websocket_frames).toBe(1); + expect(snapshot.dropped_events.total).toBe(1); +}); diff --git a/src/browser/CdpCaptureEvents.ts b/src/browser/CdpCaptureEvents.ts index 02f06cfce5..b97e114334 100644 --- a/src/browser/CdpCaptureEvents.ts +++ b/src/browser/CdpCaptureEvents.ts @@ -26,6 +26,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { rejectedSourceMapMetadataBudgetCount = 0; readonly scriptMetadataBytesById = new Map(); retainedScriptMetadataBytes = 0; + retainedEventBytes = 0; readonly executionContextFrames = new Map(); readonly network = new Map(); readonly networkRequestTimestamps = new Map(); @@ -72,6 +73,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { this.rejectedSourceMapMetadataBudgetCount = 0; this.scriptMetadataBytesById.clear(); this.retainedScriptMetadataBytes = 0; + this.retainedEventBytes = 0; this.#scriptMetadataBudgetExclusionsRecorded = false; this.executionContextFrames.clear(); this.network.clear(); From 238914974d17729bcdcc2e6cb012745d70d98a5e Mon Sep 17 00:00:00 2001 From: Wenyu Chiou <162016108+WenyuChiou@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:35:12 -0400 Subject: [PATCH 24/37] fix(analysis-view): preserve current observation paths in summaries (#7039) --- .../analysisView/AnalysisViewService.test.ts | 102 +++++++++++++++++- .../analysisView/AnalysisViewService.ts | 63 ++++++++--- .../mcp/analyzeFunctionDetail.test.ts | 67 ++++++++++-- 3 files changed, 209 insertions(+), 23 deletions(-) diff --git a/src/application/analysisView/AnalysisViewService.test.ts b/src/application/analysisView/AnalysisViewService.test.ts index 9b8476673a..84307150b5 100644 --- a/src/application/analysisView/AnalysisViewService.test.ts +++ b/src/application/analysisView/AnalysisViewService.test.ts @@ -11,7 +11,11 @@ import { import { describe, expect, it } from "vitest"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; -import { createEvidence, parseEvidence } from "../../domain/evidence.js"; +import { + createEvidence, + parseEvidence, + rebindEvidenceToTargetPath, +} from "../../domain/evidence.js"; import { inspectAnalysisViewInputSchema } from "../../domain/analysisView/analysisView.js"; import { functionDossierSchema } from "../../domain/hopperValues.js"; import { ghidraFunctionDossier } from "../../domain/ghidraValues.fixture.js"; @@ -24,6 +28,7 @@ import { inspectAnalysisView, inspectAnalysisViewValidated, summarizeRetainedAnalysis, + projectRetainedAnalysis, } from "./AnalysisViewService.js"; import type { Evidence } from "../../domain/evidence.js"; import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; @@ -39,6 +44,84 @@ import { } from "../../../tests/fixtures/browserScenarioCapture.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +it.each([false, true])( + "summarizes the selected observation path rather than older retained display metadata (history=%s)", + (history) => { + const makeParent = (path: string) => + createEvidence( + { path, format: "pe", sha256: "c".repeat(64) }, + { id: "ghidra", name: "Ghidra", version: "12.1.4" }, + { + operation: "analyze_function", + parameters: {}, + result: functionDossierSchema.parse(ghidraFunctionDossier()), + }, + ); + const previous = makeParent("/old/fixture.exe"); + const selected = makeParent("/selected/fixture.exe"); + expect(selected.evidence_id).toBe(previous.evidence_id); + const ledger = new EvidenceLedger(); + if (history) expect(ledger.record(previous).ok).toBe(true); + const summary = summarizeRetainedAnalysis(selected, { + recordEvidence: (evidence) => ledger.record(evidence), + evidenceById: (id) => ledger.forAnalysis(id), + }); + if (!summary.ok) throw summary.error; + expect(summary.value.evidence_links).toEqual([selected.evidence_id]); + expect(summary.value.subject?.local_path).toBe( + selected.subject?.local_path, + ); + expect(ledger.get(selected.evidence_id)?.subject?.local_path).toBe( + history ? previous.subject?.local_path : selected.subject?.local_path, + ); + }, +); + +it("keeps retained admission and payload ownership for a large capture when only its display path changes", async () => { + const parent = analysisViewCaptureEvidence( + analysisViewObjectDenseCapture(40_000), + ); + const ledger = new EvidenceLedger(); + expect(ledger.record(parent).ok).toBe(true); + const retained = ledger.forAnalysis(parent.evidence_id)!; + const payload = retained.normalized_result; + const subject = retained.subject!; + const selected = rebindEvidenceToTargetPath(retained, { + path: "/selected/capture.har", + sha256: subject.digest.sha256, + format: subject.format, + ...(subject.architecture === null + ? {} + : { architecture: subject.architecture }), + }); + const retention = { + recordEvidence: (evidence: Evidence) => ledger.record(evidence), + evidenceById: (id: string) => ledger.forAnalysis(id), + }; + const view = inspectAnalysisViewInputSchema.parse({ + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "api-surface" }, + }).view; + for (const evidence of [retained, selected]) { + const result = projectRetainedAnalysis(evidence, retention, view); + if (!result.ok) throw result.error; + expect(result.value.subject?.local_path).toBe(evidence.subject?.local_path); + expect(result.value.evidence_links).toEqual([parent.evidence_id]); + } + expect(ledger.forAnalysis(parent.evidence_id)).toBe(retained); + expect(retained.normalized_result).toBe(payload); + expect(retained.subject?.local_path).toBe(parent.subject?.local_path); + expect( + await inspectAnalysisView({ + source: { kind: "inline", evidence: selected }, + view: { kind: "api-surface" }, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "memory" }, + }); +}); + it.each(["layout", "javascript"])( "carries the retained %s source snapshot into repeatable shared views", async (kind) => { @@ -410,6 +493,23 @@ describe("summarizeRetainedAnalysis", () => { }), ).toEqual(err(conflict)); }); + + it("still rejects missing or mismatched retained identities after a successful writer reply", () => { + for (const [retained, reason] of [ + [undefined, "missing"], + [analysisViewLayoutEvidence(), "identity_mismatch"], + ] as const) { + expect( + summarizeRetainedAnalysis(parent, { + recordEvidence: () => ok("added"), + evidenceById: () => retained, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError", reason }, + }); + } + }); }); it("seals inferred and literal HAR pages, including a null artifact digest", async () => { diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index 55e614be31..0867b8051a 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -16,7 +16,12 @@ import { import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import { safeParseEvidence } from "../../domain/evidenceValidation.js"; -import { createEvidence, type Evidence } from "../../domain/evidence.js"; +import { + createEvidence, + rebindEvidenceToTargetPath, + type Evidence, + type EvidenceSubjectTarget, +} from "../../domain/evidence.js"; import { inspectAnalysisViewInputSchema, prepareAnalysisViewProjection, @@ -179,7 +184,20 @@ export const inspectAnalysisViewValidated = ( } const resolved = resolveAnalysisViewSource(input.source, lookup); if (!resolved.ok) return resolved; - const parent = resolved.value; + return projectResolvedAnalysisView(input, projection, resolved.value); +}; + +/** Keep payload ownership separate from this delivery's display-only subject. */ +const projectResolvedAnalysisView = ( + input: ReadyAnalysisViewInput, + projection: ReturnType, + parent: Evidence, + selectedTarget?: EvidenceSubjectTarget, +): Result => { + const presentation = + selectedTarget === undefined + ? parent + : rebindEvidenceToTargetPath(parent, selectedTarget); const projected = projection.project({ evidenceId: parent.evidence_id, operation: parent.operation, @@ -190,15 +208,15 @@ export const inspectAnalysisViewValidated = ( parent.normalized_result, // Sealing an Evidence root deep-freezes it; only the root is registered. sealedResult: isImmutableJsonSnapshot(parent), - subject: parent.subject, + subject: presentation.subject, limitations: parent.limitations, parameters: parent.parameters, artifact: - parent.subject === null + presentation.subject === null ? null : { - path: parent.subject.local_path, - sha256: parent.subject.digest.sha256, + path: presentation.subject.local_path, + sha256: presentation.subject.digest.sha256, }, }); if (!projected.ok) return projected; @@ -211,7 +229,7 @@ export const inspectAnalysisViewValidated = ( `Parent subject SHA-256 ${parent.subject.digest.sha256} differs from analysis artifact SHA-256 ${projected.value.artifact?.sha256 ?? "unknown"}.`, ), ); - return ok(createAnalysisViewEvidence(input, parent, projected.value)); + return ok(createAnalysisViewEvidence(input, presentation, projected.value)); }; /** Session Evidence ownership required to deliver a view instead of a complete record. */ @@ -254,7 +272,7 @@ export const requireAnalysisViewRetention = ( /** * Retain complete analysis Evidence in the session, then project its summary - * view from the retained record. Callers page or select the rest through + * with this observation's subject path. Callers page or select the rest through * inspect_analysis_view with the view's parent Evidence ID. */ export const summarizeRetainedAnalysis = ( @@ -273,12 +291,29 @@ export const projectRetainedAnalysis = ( if (!ready.ok) return ready; const recorded = ready.value.recordEvidence(evidence); if (!recorded.ok) return recorded; - return inspectAnalysisViewValidated( - { - source: { kind: "retained-evidence", evidence_id: evidence.evidence_id }, - view, - }, - ready.value.evidenceById, + const source = { + kind: "retained-evidence" as const, + evidence_id: evidence.evidence_id, + }; + const retained = resolveEvidenceInput(source, ready.value.evidenceById); + if (!retained.ok) return retained; + // Duplicate semantic records retain their original ledger metadata. Detail + // selection changes presentation, not the historical lookup or payload owner. + const subject = evidence.subject; + return projectResolvedAnalysisView( + { source, view }, + prepareAnalysisViewProjection(view), + retained.value, + subject === null + ? undefined + : { + path: subject.local_path, + sha256: subject.digest.sha256, + format: subject.format, + ...(subject.architecture === null + ? {} + : { architecture: subject.architecture }), + }, ); }; diff --git a/tests/boundary/mcp/analyzeFunctionDetail.test.ts b/tests/boundary/mcp/analyzeFunctionDetail.test.ts index 4cb3632f44..af674bae64 100644 --- a/tests/boundary/mcp/analyzeFunctionDetail.test.ts +++ b/tests/boundary/mcp/analyzeFunctionDetail.test.ts @@ -13,6 +13,8 @@ import { ok } from "../../../src/domain/result.js"; import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; import { parseGhidraFunctionInput } from "../../../src/ghidra/GhidraFunctionValues.js"; +import { EvidenceLedger } from "../../../src/application/investigation/EvidenceLedger.js"; +import { registerAnalysisViewTool } from "../../../src/server/registerAnalysisViewTool.js"; const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); const partialCallees = { @@ -23,13 +25,14 @@ const partialCallees = { // A fixed provider that returns the fixture dossier for any analyze_function // call, after validating the request exactly as the production Ghidra client does. -const connect = async (retain = true, result = dossier) => { - const ledger = new Map(); - const record = (evidence: Evidence) => { - ledger.set(evidence.evidence_id, evidence); - return ok("added" as const); - }; - const lookup = (id: string) => ledger.get(id); +const connect = async ( + retain = true, + result = dossier, + path = () => "/fixtures/native.exe", +) => { + const ledger = new EvidenceLedger(); + const record = (evidence: Evidence) => ledger.record(evidence); + const lookup = (id: string) => ledger.forAnalysis(id); const execute = vi.fn( async (operation: string, parameters: Record) => { if (operation !== "analyze_function") @@ -43,7 +46,7 @@ const connect = async (retain = true, result = dossier) => { limitations: [], locations: [], subject: { - path: "/fixtures/native.exe", + path: path(), format: "pe" as const, sha256: "a".repeat(64), }, @@ -70,6 +73,12 @@ const connect = async (retain = true, result = dossier) => { allowsSnapshotReplay: undefined, recordWorkflowSnapshot: undefined, }); + registerAnalysisViewTool( + server, + pino({ enabled: false }), + retain ? lookup : undefined, + retain ? record : undefined, + ); const client = new Client({ name: "fn-detail-client", version: "1" }); onTestFinished(async () => { await client.close(); @@ -82,6 +91,48 @@ const connect = async (retain = true, result = dossier) => { return { client, ledger, execute }; }; +it("keeps complete and summary delivery on the current observed path despite retained same-ID history", async () => { + let selected = "/old/native.exe"; + const { client, ledger } = await connect(true, dossier, () => selected); + const call = async (detail: "complete" | "summary") => { + const reply = await client.callTool({ + name: "analyze_function", + arguments: { procedure: "main", detail }, + }); + expect(reply.isError).not.toBe(true); + return parseEvidence(reply.structuredContent); + }; + const old = await call("complete"); + selected = "/selected/native.exe"; + const complete = await call("complete"); + expect(complete.evidence_id).toBe(old.evidence_id); + expect(complete.subject?.local_path).toBe(selected); + const summary = await call("summary"); + expect(summary.evidence_links).toEqual([complete.evidence_id]); + expect(summary.subject?.local_path).toBe(complete.subject?.local_path); + expect(summary.normalized_result).toMatchObject({ + artifact: { path: selected }, + }); + expect(summary.locations).toContainEqual({ + kind: "artifact-path", + path: selected, + }); + expect(ledger.get(complete.evidence_id)?.subject?.local_path).toBe( + "/old/native.exe", + ); + const historicalReply = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: complete.evidence_id }, + view: { kind: "summary" }, + }, + }); + expect(historicalReply.isError).not.toBe(true); + expect( + parseEvidence(historicalReply.structuredContent).subject?.local_path, + ).toBe("/old/native.exe"); +}); + it("defaults to a retained summary and strips delivery detail before the provider", async () => { const { client, ledger, execute } = await connect(); const progress: number[] = []; From 07eb044f77444de131d6c1bcdec082e5a5fcd221 Mon Sep 17 00:00:00 2001 From: Wenyu Chiou <162016108+WenyuChiou@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:35:27 -0400 Subject: [PATCH 25/37] fix(snapshot): decline ambiguous exact evidence replay (#7037) --- src/domain/analysisSnapshot.test.ts | 45 +++++++++ src/domain/analysisSnapshot.ts | 39 +++++--- .../composedWorkflowEvidenceLinks.test.ts | 99 +++++++++++++++++++ 3 files changed, 169 insertions(+), 14 deletions(-) diff --git a/src/domain/analysisSnapshot.test.ts b/src/domain/analysisSnapshot.test.ts index 841585bf86..e72c0740a3 100644 --- a/src/domain/analysisSnapshot.test.ts +++ b/src/domain/analysisSnapshot.test.ts @@ -422,6 +422,51 @@ describe("analysis snapshot canonical ordering and scope", () => { }); }); +it("does not select arbitrary provenance when multiple authenticated records match one replay entry", () => { + const { current, snapshot } = snapshotWithHistoricalEvidence(); + const source = snapshot.evidence_bundle.records.find( + (record) => record.operation === "unprofiled_query", + )!; + const alternative = createEvidence( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROVIDER, + { + operation: current.operation, + parameters: current.parameters, + result: current.normalized_result, + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + evidenceLinks: [source.evidence_id], + }, + ); + expect(alternative.evidence_id).not.toBe(current.evidence_id); + const ambiguous = parseAnalysisSnapshot({ + ...snapshot, + evidence_bundle: createEvidenceBundle([ + ...snapshot.evidence_bundle.records, + alternative, + ]), + }); + expect(ambiguous.entries).toEqual(snapshot.entries); + const reversedInsertion = parseAnalysisSnapshot({ + ...ambiguous, + evidence_bundle: createEvidenceBundle( + [...ambiguous.evidence_bundle.records].reverse(), + ), + }); + expect(reversedInsertion).toEqual(ambiguous); + for (const candidate of [ambiguous, reversedInsertion]) + expect( + snapshotEvidenceForQuery(candidate, { + target: ANALYSIS_SNAPSHOT_TARGET, + bindingProfile: ANALYSIS_SNAPSHOT_PROFILE, + operation: current.operation, + parameters: current.parameters, + provider: ANALYSIS_SNAPSHOT_PROVIDER, + evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE, + }), + ).toBeUndefined(); +}); + describe("analysis snapshot Evidence binding", () => { it("finds only Evidence committed to the exact binding and profile", () => { const { current, snapshot } = snapshotWithHistoricalEvidence(); diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index 12c1b6cf9b..3d5d7df395 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -502,20 +502,31 @@ export const snapshotEvidenceForQuery = ( (candidate) => candidate.query_id === queryId, ); if (entry === undefined && workflowEntry === undefined) return undefined; - const evidence = snapshot.evidence_bundle.records.find( - (record) => - record.subject?.digest.sha256 === target.sha256 && - record.operation === operation && - record.provider.id === provider.id && - record.provider.name === provider.name && - record.provider.version === provider.version && - record.analysis_profile !== null && - analysisProfilesEqual(record.analysis_profile, evidenceProfile) && - canonicalJson(record.parameters) === encodedParameters && - ((entry !== undefined && evidenceMatchesEntry(record, entry, snapshot)) || - (workflowEntry !== undefined && - evidenceMatchesWorkflowEntry(record, workflowEntry, snapshot))), - ); + let evidence: Evidence | undefined; + for (const record of snapshot.evidence_bundle.records) { + if ( + !( + record.subject?.digest.sha256 === target.sha256 && + record.operation === operation && + record.provider.id === provider.id && + record.provider.name === provider.name && + record.provider.version === provider.version && + record.analysis_profile !== null && + analysisProfilesEqual(record.analysis_profile, evidenceProfile) && + canonicalJson(record.parameters) === encodedParameters && + ((entry !== undefined && + evidenceMatchesEntry(record, entry, snapshot)) || + (workflowEntry !== undefined && + evidenceMatchesWorkflowEntry(record, workflowEntry, snapshot))) + ) + ) + continue; + // Entries bind the result, not every part of the Evidence commitment + // (for example, upstream links). Do not choose provenance by ID sort order. + if (evidence !== undefined && evidence.evidence_id !== record.evidence_id) + return undefined; + evidence = record; + } return evidence === undefined ? undefined : rebindEvidenceToTargetPath(evidence, target); diff --git a/tests/boundary/filesystem/composedWorkflowEvidenceLinks.test.ts b/tests/boundary/filesystem/composedWorkflowEvidenceLinks.test.ts index c31c3f5662..11be0d2eea 100644 --- a/tests/boundary/filesystem/composedWorkflowEvidenceLinks.test.ts +++ b/tests/boundary/filesystem/composedWorkflowEvidenceLinks.test.ts @@ -6,6 +6,105 @@ import { expect, it } from "vitest"; import { runDirectAnalysis } from "../../../src/application/DirectAnalysis.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { composedWorkflowFixture } from "../../fixtures/composedWorkflowEvidence.js"; +import { readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; + +it.each([0, 1, 12])( + "does not replay an arbitrary historical workflow provenance when the snapshot binding is ambiguous (%i procedures)", + async (procedureCount) => { + const original = await composedWorkflowFixture(); + original.summary.procedure_count = procedureCount; + const target = await original.open(); + const snapshotPath = join(dirname(original.path), "analysis.json"); + const current = parseEvidence( + await runDirectAnalysis( + original.dependencies, + original.path, + "binary_overview", + {}, + { snapshotPath }, + ), + ); + expect(current.evidence_links).toHaveLength(1); + const snapshot = parseAnalysisSnapshot( + JSON.parse(await readFile(snapshotPath, "utf8")), + ); + const uniqueReplay = await composedWorkflowFixture(); + expect(uniqueReplay.session.importAnalysisSnapshot(snapshot).ok).toBe(true); + expect(uniqueReplay.session.importAnalysisSnapshot(snapshot)).toEqual( + ok(0), + ); + const unique = parseEvidence( + await runDirectAnalysis( + uniqueReplay.dependencies, + original.path, + "binary_overview", + {}, + { snapshotPath }, + ), + ); + expect(unique.evidence_id).toBe(current.evidence_id); + expect(unique.evidence_links).toEqual(current.evidence_links); + expect(uniqueReplay.calls).toEqual([]); + // Earlier composed observations did not carry links. Both are authentic, + // but the serialized replay binding does not identify which provenance won. + const historical = createEvidence(target, current.provider, { + operation: current.operation, + parameters: current.parameters, + result: current.normalized_result, + rawResult: current.raw_result, + analysisProfile: current.analysis_profile!, + confidence: "derived", + limitations: current.limitations, + }); + expect(historical.evidence_links).toEqual([]); + expect(historical.evidence_id).not.toBe(current.evidence_id); + const ambiguous = parseAnalysisSnapshot({ + ...snapshot, + evidence_bundle: createEvidenceBundle([ + ...snapshot.evidence_bundle.records, + historical, + ]), + }); + expect(ambiguous.workflow_entries).toEqual(snapshot.workflow_entries); + await writeFile(snapshotPath, JSON.stringify(ambiguous)); + const replay = await composedWorkflowFixture(); + const delivered = parseEvidence( + await runDirectAnalysis( + replay.dependencies, + original.path, + "binary_overview", + {}, + { snapshotPath }, + ), + ); + expect(delivered.normalized_result).toEqual(current.normalized_result); + expect([current.evidence_id, historical.evidence_id]).not.toContain( + delivered.evidence_id, + ); + expect(delivered.evidence_links).toEqual(current.evidence_links); + expect( + delivered.limitations.some((limitation) => + limitation.includes("this call did not re-run the provider"), + ), + ).toBe(true); + expect(replay.calls).toEqual([]); + const refreshedReplay = await composedWorkflowFixture(); + const refreshed = parseEvidence( + await runDirectAnalysis( + refreshedReplay.dependencies, + original.path, + "binary_overview", + {}, + { snapshotPath }, + ), + ); + expect(refreshed.evidence_id).toBe(delivered.evidence_id); + expect(refreshed.evidence_links).toEqual(delivered.evidence_links); + expect(refreshedReplay.calls).toEqual([]); + }, +); it.each([false, true])( "links a profiled binary overview only to the atomic record it consumed (cached=%s)", From efab65c2f93af770ce686097d17fd1cb01dc66ce Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:48:43 +0800 Subject: [PATCH 26/37] docs: codify boundary regression checks Turn recurring parser and projection failures into a selectable regression guide covering format assumptions, raw identity, missing values, partial work, independent limits, representation expansion, snapshots, and public delivery. Link it from the agent instructions and PR template without adding checkboxes. Clarify completed-observation ownership, digest representations, and exclusive success/failure budget accounting. Preserve capacity fixtures and use byte comparisons for large buffers to avoid diagnostic allocation blowups. --- .github/pull_request_template.md | 4 ++++ AGENTS.md | 2 +- docs/testing.md | 28 ++++++++++++++++++++++++++++ docs/tool-design.md | 27 +++++++++++++++++++++++---- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3c6c0ca03e..69e4d62cda 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -68,6 +68,10 @@ source, link, or rendered-document checks. --> - User-visible CLI/MCP output (if applicable): - Remaining proof gaps: +For boundary fixes, identify the applicable [boundary regressions](https://github.com/morluto/rea/blob/main/docs/testing.md#boundary-regressions) +and [lifecycle regressions](https://github.com/morluto/rea/blob/main/docs/testing.md#lifecycle-regressions) +covered by the evidence above, and explain relevant gaps. + For evidence-bearing changes: - [ ] Observed, derived, and inferred claims remain distinguishable. diff --git a/AGENTS.md b/AGENTS.md index b542f3cfa2..972f03da73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,7 +80,7 @@ Coordinate expensive checks across agents. If a test unexpectedly consumes heavy For tool design, follow [docs/tool-design.md](docs/tool-design.md): start from the analyst question, reuse existing contracts, preserve caller choices, and return useful evidence inline. Limits must follow real format, protocol, authority, or resource constraints. Account for representation expansion before allocation; derive completeness from examined coverage. -For test changes or provider claims, use [docs/testing.md](docs/testing.md). Prefer real public CLI/MCP workflows, then production-boundary integration, then producer goldens. Keep module tests for distinct semantics stronger workflows cannot reliably reproduce. Preserve malformed-input, cancellation, cleanup, permission, capacity, and target-identity coverage when pruning. Report unverified coverage explicitly; simulated providers do not establish real-provider behavior. +For test changes or provider claims, use [docs/testing.md](docs/testing.md). Select applicable [boundary regressions](docs/testing.md#boundary-regressions) for parser, inventory, snapshot, and projection fixes. Prefer real public CLI/MCP workflows, then production-boundary integration, then producer goldens. Keep module tests for distinct semantics stronger workflows cannot reliably reproduce. Preserve malformed-input, cancellation, cleanup, permission, capacity, and target-identity coverage when pruning. Report unverified coverage explicitly; simulated providers do not establish real-provider behavior. Use Conventional Commit subjects and PR titles. Mark breaking changes with `!` or a `BREAKING CHANGE:` footer. Describe behavior changes, relevant validation, and real-provider verification scope. diff --git a/docs/testing.md b/docs/testing.md index c3a5e991e4..043a1c2a71 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -200,6 +200,10 @@ work from fixture generation, schema compilation, subprocess startup and cleanup Choose elapsed, CPU and process-family memory limits before the run; a worker count or test timeout alone does not enforce those limits. +Keep capacity fixtures intact when diagnosing a test failure. For byte equality +of large buffers, use `Buffer.equals` or `Buffer.compare`; a generic structural +diff can allocate far more than the production operation. + Remove demonstrated repeated work while preserving distinct assertions and resource ownership. Share immutable fixture preparation where safe; keep fresh sessions when lifecycle or concurrency is the behavior under test. Compare the @@ -335,6 +339,30 @@ that a real loader accepts the artifact. If multiple producer revisions share a format identifier but differ in behavior, record that ambiguity rather than using one installed version as an oracle for all of them. +### Boundary regressions + +For parser, inventory, snapshot, or projection fixes, select the applicable +cases below. Assert the producing boundary's facts and their meaning in affected +consumers. Add [lifecycle regressions](#lifecycle-regressions) when resource +ownership or finalization changes; a single malformed fixture cannot establish +the whole contract. + +| Failure class | Regression that catches it | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Format assumptions | Pair valid padding, offsets, or names containing delimiters with malformed input. Identify the producer revision when behavior varies by version; an unrelated runtime API's range is not an on-disk format limit. | +| Raw identity and normalization | Exercise allowed empty or unusual source names/metadata keys and values that normalize to one lookup or display value. Preserve admitted raw spelling under the existing redactions for transport credentials and explicitly marked sensitive values. Distinguish local paths, file URLs and HTTP paths; assert source identity through consumers. | +| Missing, empty, and unknown values | Distinguish omitted, null, empty, zero, and false where the format admits them. Pair a malformed optional field with usable sibling facts; assert uncertainty without inventing absence or discarding the row. | +| Completed work followed by failure | Complete one input, then fail or cancel a later input. Assert earlier digests, metadata, and parsed facts survive with the later failure reason and accurate processed/unprocessed coverage. For cancellation, assert the exact caller abort reason; where a primary failure was already produced, assert it survives. | +| Independent limits and failures | Trigger a sample/output limit alongside a read or parse failure. Assert both dimensions survive; reaching a limit does not erase sampling failures or prove complete coverage. | +| Representation expansion and capacity | Keep a valid set near the limit and a large malformed input followed by a valid input. Cover JSON escaping, expanded paths or decoded rows as applicable. Assert serialized bounds, retained identities, later valid results, and unknown omissions without charging discarded interpretations as retained output. | +| Snapshots, caches, and comparison | Vary metadata that affects the result while keeping payload bytes equal. Exercise historical records with newly introduced fields omitted, current caller inputs on replay, and reuse across differing selections. Assert correct invalidation or unknown coverage, current request provenance, and unchanged saved Evidence and IDs. | +| Public success and error delivery | Exercise affected public CLI/MCP paths, including partial Evidence on failure. Assert retained facts and named contracts; retrieve partial Evidence when MCP returns a reference. If SDK cancellation suppresses the reply, check server settlement and any retained Evidence. An error-code assertion alone misses lost observations. | + +When producer order is unspecified, synchronize fixtures on the observed request +or completion event. A filename, generated key, or fixed delay does not prove +which item completed before cancellation. Preserve real producer ordering and +use a controlled seam for the phase that needs deterministic failure. + Packed bridge validation uses Python's standard-library AST and tar readers with `-I -S`: caller-local modules, `PYTHONPATH`, and site startup hooks must not replace those readers or execute during validation. Controlled `ast.py` diff --git a/docs/tool-design.md b/docs/tool-design.md index 1fb7e2d901..df741d1888 100644 --- a/docs/tool-design.md +++ b/docs/tool-design.md @@ -85,6 +85,17 @@ satisfy. A plan without execution does not establish runtime behavior. ### Interpret boundary facts once +The format or protocol reader owns interpretation. Pass its parsed facts and +explicit uncertainty to consumers; do not make adapters reparse display text, +reconstruct omitted values, or infer success from missing diagnostics. + +A completed read can establish byte identity even when decoding or format +validation fails. State which representation a digest identifies, such as file +bytes, Fetch response-body bytes, or UTF-8 text. Preserve admitted source values +separately from normalized lookup and display values. When the retention +contract keeps only a digest and size, state that the bytes or text are +unavailable; the digest does not establish format validity. + Comparison needs separate proofs for lookup, correspondence, content, and coverage. A hash cannot supply the proof that two retained values belong to corresponding objects. @@ -103,10 +114,6 @@ Complete traversal or inventory does not make unavailable values equal. A partial aggregate fingerprint difference cannot establish modification when omissions alone explain it; independently proven differences still survive. -The format or protocol reader owns interpretation. Pass its parsed facts and -explicit uncertainty to consumers; do not make adapters reparse display text, -reconstruct omitted values, or infer success from missing diagnostics. - Keep format recognition separate from provider admission. A recognizable header can supply useful observations while later structure remains unresolved or a provider cannot accept the artifact. Report the reason at the affected boundary @@ -133,6 +140,12 @@ establish each transition. Include temporary files and one-shot commands, as well as persistent engine sessions. Use the existing provider contracts; providers need not expose identical state machines. +In multi-stage work, retain each completed, admitted observation before awaiting +the next stage. Later reads, validation, cancellation, or finalization can fail +without erasing those facts. Carry them through typed failure outcomes with +explicit coverage of unperformed work; do not construct a successful result for +stages that never ran. + - Keep target admission, provider readiness, active work, observed process exit, and resource release separate. A valid process handle, successful launcher, or selected target establishes only its own fact. Publish available identity @@ -259,6 +272,12 @@ completeness from examined records and exhaustiveness, rather than diagnostic prose or an empty failure list. Preserve observations independently of cleanup success. +Price mutually exclusive success and failure representations separately. +Replace a reserved failure envelope when pricing success, and release discarded +decoding state from retained-output accounting. Retaining additional failed-body +text changes the capacity available to later results; treat that as a contract +change and cover its effect on later valid items. + For example, if ten applicable records are discovered, eight are examined, and five are returned, the five returned records do not establish complete coverage of the ten. Keep discovered, applicable, examined, and retained scope distinct. From 9140deae63c512c28e7a9a488cad0a053bfcdb0c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:50:18 +0800 Subject: [PATCH 27/37] docs: clarify ownership and cleanup review guidance Clarify structural parsing, admitted facts and freshness, useful facade and compatibility responsibilities, immutable JSON proof, and partial Evidence retention. Document fixture cleanup ownership, semantic fault injection and the limits of static checks; extend the existing PR owner hint. Correct the session-record architecture label. Validation: docs:check (including nine README consistency tests), rendered docs:build with link validation, touched Markdown formatting and git diff --check passed. Source and installed Vitest cleanup behavior were reviewed. No runtime behavior changed. --- .github/pull_request_template.md | 3 ++ CONTRIBUTING.md | 24 +++++++++++++-- docs/architecture.mermaid | 4 +-- docs/testing.md | 50 +++++++++++++++++++++++++------- docs/tool-design.md | 34 ++++++++++++++++++++-- 5 files changed, 98 insertions(+), 17 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 69e4d62cda..91063384cc 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -36,6 +36,9 @@ unchecked. Maintainers can help resolve open questions. Use "none" or "not applicable" only when established. --> - Semantic owner and earliest changed stage: + - Contribution scope fit and linked maintainer decision: - CLI and MCP/tool-catalog contract: - Provider, bridge, target-format, or platform compatibility: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e3ff9f31f4..ad5236520e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,18 @@ acquiring targets; optional adapter failures must preserve successful peers. Run `npm run verify:test-discovery` after adding or moving tests. Resolved import boundaries are checked by `npm run verify:module-boundaries`, included in lint. +Before retiring an alias or facade, trace runtime registration, dynamic imports, +JavaScript verifiers, subprocess scripts, and packaged entrypoints as well as +TypeScript references. Tests may be the last consumers of a superseded internal +path; migrate their distinct assertions to the current production path before +removing it. Preserve validation, lazy loading, caller policy, and resource +acquisition and cleanup supplied by the old boundary. Published APIs, persisted +formats, and independently versioned bridges require an explicit compatibility +decision; an internal representation alone does not establish that obligation. +See [interpreting boundary facts once](docs/tool-design.md#interpret-boundary-facts-once) +for admission ownership and the distinction between repeated validation and +independent checks. + For analysis workflows and agent interfaces, follow the [performance priorities](AGENTS.md#performance). Explain the selected output scope, reuse/cache decisions, and relevant before/after measurements when these @@ -372,9 +384,17 @@ Real-provider execution is uncached. ### Temporary files and diagnostics Tests use `createTestTempDirectory` from `tests/fixtures/temporaryDirectory.ts` -for exact-path, awaited cleanup, including failures and timeouts. +for exact-path, awaited cleanup, including failures and timeouts. It registers +deletion with Vitest's `onTestFinished`; do not also add its result to a roots +array or an `afterEach` removal. `createTestWorkspace` from +`tests/support/workspace/workspaceFixture.ts` leaves cleanup with its caller, +which must await `removeTestWorkspace`. Keep explicit resource teardown before +directory deletion, using the case's `finally` or `afterEach`; Vitest runs +`afterEach` before `onTestFinished` and awaits finish hooks in reverse +registration order. `npm run verify:test-temp-hygiene` runs the suite under a fresh `TMPDIR` and -rejects remaining REA-owned paths. Never glob-clean shared `/tmp/rea-*` content. +rejects remaining REA-owned paths. It cannot detect duplicate deletion owners or +prove resource-close ordering. Never glob-clean shared `/tmp/rea-*` content. `REA_LOG_LEVEL` selects `trace`, `debug`, `info`, `warn`, `error`, `fatal`, or `silent`. Logs go to stderr: MCP defaults to `info`, and one-shot CLI logging diff --git a/docs/architecture.mermaid b/docs/architecture.mermaid index d38bd957f2..015b835ecb 100644 --- a/docs/architecture.mermaid +++ b/docs/architecture.mermaid @@ -23,7 +23,7 @@ graph TD SESSION["SessionProviderRouter + BinarySession
one immutable deep binding per target
disjoint auxiliary routing"] REGISTRY["AnalysisProviderRegistry
sorted deep candidates
deterministic selection, no fallback"] RECORDS["src/application/investigation/
composed InvestigationRecords + narrow ports
EvidenceLedger and Unknown ownership"] - SNAPSHOT["BinarySessionRecords + AnalysisSnapshotCache
binary target/profile snapshots
compatibility facade and post-commit notifications"] + SNAPSHOT["BinarySessionRecords + AnalysisSnapshotCache
binary target/profile snapshots and session file ownership
record coordination and post-commit notifications"] JS_RECON["src/application/javascript/ + src/artifacts/javascript/
JavaScript artifact reconstruction
directory/ASAR reads
AST-only bundle projection"] end @@ -75,7 +75,7 @@ graph TD end subgraph "Domain Layer" - DOMAIN["src/domain/
Pure side-effect-free modules
evidence, comparisons, application graphs"] + DOMAIN["src/domain/
Pure side-effect-free modules
format declarations, Evidence, comparisons, application graphs"] end subgraph "Configuration" diff --git a/docs/testing.md b/docs/testing.md index 043a1c2a71..b7b8f0750d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -140,6 +140,12 @@ static tasks. Both lint caches include the website, documentation configuration, and experiment code scanned by the checker; a regression compares the actual lint inventory with Turbo's resolved inputs. +Static checks locate candidates; they do not establish semantic ownership. +`jscpd` finds token clones, and Knip counts test references as uses. Module +boundaries and architecture guards enforce their declared import and source +rules. Passing them does not prove that two parsers agree, a wrapper is still +needed at runtime, or cleanup has one owner. + Routine iterations and rebases need focused regressions and relevant checks. Before handing off a PR, run `npm run check` and generated-document checks when applicable; CI owns the full suite and coverage. Use the full local gate for @@ -169,6 +175,14 @@ with the consumer preloaded as well as alone. For simulated protocol traces, use an acknowledged event barrier before asserting complete observations; a zero-duration capture does not guarantee queued events have arrived. +Place faults at the semantic boundary under test: the selected read, EOF, hash +completion, or cleanup attempt. Global call ordinals can shift during setup or +refactoring without reaching that phase. For controlled asynchronous gates, use +`awaitAcknowledgement` from `tests/fixtures/lifecycleAcknowledgement.ts` to race +the acknowledgement against triggering-operation settlement and a bounded +deadline. Register release gates before launch, then release and join launched +work in `finally` before restoring mocks, even when the barrier fails. + Focused contract tests should import the named contract family they exercise; reserve the full tool catalog for catalog-wide assertions. Resolve contracts by operation name rather than array position. @@ -312,6 +326,13 @@ Trivial helper assertions need no replacement. Remove abandoned production scaffolding only after checking runtime and verifier consumers, including compiled imports. +After an owner changes, move fault injection and event spies to the path the +production consumer actually uses, and assert that the intended boundary was +reached. Keep independent expectations from authored input, producer output, or +an external oracle; comparing two paths through the same replacement algorithm +does not preserve the old oracle. A negative probe inside a conditional or loop +must assert that it ran. + Use representative scenarios rather than Cartesian matrices of equivalent cases. Share setup without hiding inputs or expected evidence. For successful `Result` values, throw the typed error before asserting the value; a preceding @@ -358,6 +379,10 @@ the whole contract. | Snapshots, caches, and comparison | Vary metadata that affects the result while keeping payload bytes equal. Exercise historical records with newly introduced fields omitted, current caller inputs on replay, and reuse across differing selections. Assert correct invalidation or unknown coverage, current request provenance, and unchanged saved Evidence and IDs. | | Public success and error delivery | Exercise affected public CLI/MCP paths, including partial Evidence on failure. Assert retained facts and named contracts; retrieve partial Evidence when MCP returns a reference. If SDK cancellation suppresses the reply, check server settlement and any retained Evidence. An error-code assertion alone misses lost observations. | +For partial Evidence delivery, verify that same-session readback preserves +identity and provenance without rerunning the provider. Include a no-writer +control that keeps completed Evidence inline and preserves the original failure. + When producer order is unspecified, synchronize fixtures on the observed request or completion event. A filename, generated key, or fixed delay does not prove which item completed before cancellation. Preserve real producer ordering and @@ -412,15 +437,16 @@ the affected owners and applicable scenarios below. Include temporary snapshots and one-shot commands. Use public CLI/MCP workflows where practical and a controlled production seam for interleavings that cannot be reproduced reliably. -| Scenario | Assertion and control | -| --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Target admitted; startup still pending | Status preserves the readiness distinction and observed startup activity; readiness requires the provider's documented proof. | -| First handshake is late, absent, invalid, or recovered during close | Initial and recovery paths retain the same authenticated identity facts. Foreign run/token controls cannot authorize cleanup; authenticated idle/native work survives the unauthenticated listener deadline. | -| Launch returns an error with a valid process handle | Cover already-exited and still-live owned processes. Verify observed exit separately from creation; failed termination or unavailable identity remains unresolved. | -| Cancellation arrives after workload exit, during result reads or temporary-root removal | Return typed cancellation with available partial observations and actual cleanup status. The non-aborted control succeeds; existing primary/cleanup failures keep their precedence. | -| Children have mixed health or cleanup outcomes | Missing peer health does not erase an observed failure or activity. Idle plus unknown stays unknown. Retry a failed child without closing a successful peer again, including after a close-option change. | -| Cleanup fails after success or primary failure | Public output retains the completed observation or primary diagnostic, remaining resources, and owner recovery action. Restore the denied resource and verify the owning cleanup retry. | -| Target exits without an acknowledgement, or transport disappears while it stays alive | Positive owned exit proof resolves target lifetime; other resources still need cleanup. Lost transport alone does not prove exit. Verify unrelated/attached targets survive and same-connection recovery works when supported. | +| Scenario | Assertion and control | +| --------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Target admitted; startup still pending | Status preserves the readiness distinction and observed startup activity; readiness requires the provider's documented proof. | +| First handshake is late, absent, invalid, or recovered during close | Initial and recovery paths retain the same authenticated identity facts. Foreign run/token controls cannot authorize cleanup; authenticated idle/native work survives the unauthenticated listener deadline. | +| Launch returns an error with a valid process handle | Cover already-exited and still-live owned processes. Verify observed exit separately from creation; failed termination or unavailable identity remains unresolved. | +| Cancellation arrives after workload exit, during result reads or temporary-root removal | Return typed cancellation with available partial observations and actual cleanup status. The non-aborted control succeeds; existing primary/cleanup failures keep their precedence. | +| Children have mixed health or cleanup outcomes | Missing peer health does not erase an observed failure or activity. Idle plus unknown stays unknown. Retry a failed child without closing a successful peer again, including after a close-option change. | +| Cleanup fails after success or primary failure | Public output retains the completed observation or primary diagnostic, remaining resources, and owner recovery action. Restore the denied resource and verify the owning cleanup retry. | +| Cleanup owners recover after an error was returned | Retain the original child and aggregate errors and project them after each owner releases. A successful peer stays released; the aggregate remains incomplete until all owners release. Preserve the first cause even when rejection is `undefined`. | +| Target exits without an acknowledgement, or transport disappears while it stays alive | Positive owned exit proof resolves target lifetime; other resources still need cleanup. Lost transport alone does not prove exit. Verify unrelated/attached targets survive and same-connection recovery works when supported. | Shared process-supervision controls separate leader exit from output closure. Cover output during held or failed cleanup, bounded retention and drain timeout, @@ -587,7 +613,11 @@ creation outside the workspace seam and its narrowly documented boundary/package exceptions. The shared temporary-directory and workspace helpers preserve caller labels -under an owned `rea-` prefix when needed. `npm run verify:test-temp-hygiene` +under an owned `rea-` prefix when needed. Their +[cleanup ownership differs](https://github.com/morluto/rea/blob/main/CONTRIBUTING.md#temporary-files-and-diagnostics): +`createTestTempDirectory` registers `onTestFinished` deletion, while +`createTestWorkspace` requires caller cleanup. Close resources before removing +their directory. `npm run verify:test-temp-hygiene` uses the same ownership rule to find retained fixtures under a private temporary root. Third-party caches remain outside that ownership rule. diff --git a/docs/tool-design.md b/docs/tool-design.md index df741d1888..23f1202548 100644 --- a/docs/tool-design.md +++ b/docs/tool-design.md @@ -88,6 +88,10 @@ satisfy. A plan without execution does not establish runtime behavior. The format or protocol reader owns interpretation. Pass its parsed facts and explicit uncertainty to consumers; do not make adapters reparse display text, reconstruct omitted values, or infer success from missing diagnostics. +Client registration inspections, for example, carry command identity and +activation independently of readiness. An owned registration may be disabled +or stale; unresolved ownership remains unknown. Consumers use those facts +instead of reclassifying the raw command. A completed read can establish byte identity even when decoding or format validation fails. State which representation a digest identifies, such as file @@ -220,6 +224,12 @@ stages that never ran. applies rather than duplicate close/finally handling. - Give descriptor-backed reads an owner before opening the file. A rejected close must retain the exact handle and preserve any earlier failure. + Carry [`openRegularFile`](https://github.com/morluto/rea/blob/main/src/filesystem/RegularFile.ts)'s + admitted handle and initial metadata rather than restatting only to rebuild + those facts. Keep fresh descriptor and selected-path identity checks after + work, comparing against the original baseline instead of replacing it with + a later stat. Preserve explicit bigint reads when ordinary admission metadata + cannot supply the required size or timestamp precision. Stable file reads expose completed values separately from cancellation so consumers retain acquired identity facts after successful close or cleanup retry. If an abortable caller stops waiting, keep the underlying work in that owner's @@ -250,9 +260,13 @@ through the producing owner and affected CLI/MCP consumers. ## Implement and evaluate -Put provider protocols/parsing in adapters and shared CLI/MCP workflows in the -application layer. Share Evidence provenance, unknown projection, and session -retention semantics; matching payloads alone do not establish parity. +Put provider wire-protocol parsing in adapters and shared CLI/MCP workflows in +the application layer. Reusable format declarations may live in pure domain +owners: [`peOptionalHeader.ts`](https://github.com/morluto/rea/blob/main/src/domain/peOptionalHeader.ts) +decodes PE structure, while consumers retain machine admission, directory-address +interpretation, and file/mapping bounds. Share Evidence provenance, unknown +projection, and session retention semantics; matching payloads alone do not +establish parity. Let `createEvidence` own JSON validation and snapshotting of ordinary observation payloads. `EvidenceObservationInput` accepts semantically parsed parameters, @@ -264,6 +278,20 @@ Evidence envelope. `EvidenceObservation` retains the stronger JSON contract used by producer helpers and `createImmutableEvidenceSteps`. The cooperative constructor also requires an authenticated immutable result; preserve that ownership proof. +For objects, proof comes from completed sealing by +[`immutableJson.ts`](https://github.com/morluto/rea/blob/main/src/domain/immutableJson.ts); +`readonly` or `Object.freeze` alone does not establish it. JSON primitives are +directly eligible. Seal owned data and complete cooperative sealing before +reusing its result. + +When a failed or cancelled operation retains completed, valid partial Evidence, +use the shared [`ToolResultDelivery`](https://github.com/morluto/rea/blob/main/src/server/ToolResultDelivery.ts) +path with the session's Evidence writer. Keep the original failure and the +named contract's target-content marking; replace the inline partial with a +retained reference only after recording succeeds. An absent writer or a writer +returning a failure `Result` leaves the partial inline, as does a partial that +is not valid Evidence. Unexpected recorder exceptions propagate. An `evidence_id` +member alone does not establish Evidence validity or retention. Measure the resource before choosing a budget and enforce it before the costly step. Account for value alternatives, trace-frame products, retained records, From f2fa951c001e90f59ac1f76db45a7ea3fd7b2cdc Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:30:50 +0800 Subject: [PATCH 28/37] fix(cancellation): retain caller reasons across digest and startup --- .../reverse/ReverseEngineeringService.ts | 24 ++++++++++--- src/cli/environmentCommands.ts | 27 +++++++++----- src/gdb/GdbSessionManager.ts | 24 ++++++++++--- .../GdbSessionManagerCoverage.test.ts | 8 +++-- tests/composition/reverseEngineering.test.ts | 36 +++++++++++++++++++ 5 files changed, 99 insertions(+), 20 deletions(-) diff --git a/src/application/reverse/ReverseEngineeringService.ts b/src/application/reverse/ReverseEngineeringService.ts index 23a9613f77..68d5cb6c02 100644 --- a/src/application/reverse/ReverseEngineeringService.ts +++ b/src/application/reverse/ReverseEngineeringService.ts @@ -1057,7 +1057,11 @@ export const digestFile = async ( remainingMs, ); const abort = (): void => { - stream.destroy(new AnalysisCancelledError("artifact_digest")); + stream.destroy( + new AnalysisCancelledError("artifact_digest", { + cause: signal?.reason, + }), + ); }; const cleanup = (): void => { clearTimeout(timer); @@ -1066,7 +1070,11 @@ export const digestFile = async ( if (signal?.aborted) { cleanup(); stream.destroy(); - rejectDigest(new AnalysisCancelledError("artifact_digest")); + rejectDigest( + new AnalysisCancelledError("artifact_digest", { + cause: signal.reason, + }), + ); return; } signal?.addEventListener("abort", abort, { once: true }); @@ -1139,7 +1147,9 @@ const statWithDeadline = ( timeoutMs: number, ): Promise => { if (signal?.aborted) - return Promise.reject(new AnalysisCancelledError("artifact_digest")); + return Promise.reject( + new AnalysisCancelledError("artifact_digest", { cause: signal.reason }), + ); return new Promise((resolveStat, rejectStat) => { let settled = false; const finish = (outcome: Result): void => { @@ -1151,7 +1161,13 @@ const statWithDeadline = ( else rejectStat(outcome.error); }; const abort = (): void => - finish(err(new AnalysisCancelledError("artifact_digest"))); + finish( + err( + new AnalysisCancelledError("artifact_digest", { + cause: signal?.reason, + }), + ), + ); const timer = setTimeout( () => finish( diff --git a/src/cli/environmentCommands.ts b/src/cli/environmentCommands.ts index 6445523de5..abce912196 100644 --- a/src/cli/environmentCommands.ts +++ b/src/cli/environmentCommands.ts @@ -124,8 +124,12 @@ interface ServeInputs { readonly served: ServedState; } -const cancelledServe = (): JsonValue => - cliError(new AnalysisCancelledError(CLI_COMMANDS.environmentServe)); +const cancelledServe = (signal: AbortSignal): JsonValue => + cliError( + new AnalysisCancelledError(CLI_COMMANDS.environmentServe, { + cause: signal.reason, + }), + ); /** Pure replay, as `--replay-only` asks for: no state is derived. */ const REPLAY_ONLY: ServedState = { @@ -149,14 +153,14 @@ const loadServeInputs = async ( if (!evidence.ok) return evidence; const { loadEnvironmentSource } = await import("../application/environment/environmentSource.js"); - if (signal.aborted) return err(cancelledServe()); + if (signal.aborted) return err(cancelledServe(signal)); const source = loadEnvironmentSource(manifest.value, evidence.value); if (!source.ok) return err(cliError(source.error)); if (input.replayOnly) return ok({ source: source.value, served: REPLAY_ONLY }); const { prepareServedState } = await import("../application/environment/environmentState.js"); - if (signal.aborted) return err(cancelledServe()); + if (signal.aborted) return err(cancelledServe(signal)); return ok({ source: source.value, served: prepareServedState(source.value, evidence.value, { @@ -172,7 +176,7 @@ const serveEnvironmentCommand = async ( logger: Logger, signal: AbortSignal, ): Promise => { - if (signal.aborted) return cancelledServe(); + if (signal.aborted) return cancelledServe(signal); const endpoint = parseMcpHttpEndpoint(environment, { ...(input.host === undefined ? {} : { host: input.host }), ...(input.port === undefined ? {} : { port: input.port }), @@ -192,13 +196,13 @@ const serveEnvironmentCommand = async ( { code: report.code, error: report.message }, "Environment state unavailable; serving recorded replay", ); - if (signal.aborted) return cancelledServe(); + if (signal.aborted) return cancelledServe(signal); // Loaded here so registering the command catalog never loads the HTTP listener. const { serveEnvironment } = await import("../server/environment/serveEnvironment.js"); const { StreamableHttpListenError } = await import("../server/http/streamableHttpListenError.js"); - if (signal.aborted) return cancelledServe(); + if (signal.aborted) return cancelledServe(signal); // A listener that fails after binding ends the command instead of serving nothing. let failListener: (error: StreamableHttpListenError) => void = () => undefined; @@ -221,7 +225,8 @@ const serveEnvironmentCommand = async ( if (!started.ok) return cliError(started.error); served = started.value; } catch (cause: unknown) { - if (signal.aborted && cause === signal.reason) return cancelledServe(); + if (signal.aborted && cause === signal.reason) + return cancelledServe(signal); if (!(cause instanceof StreamableHttpListenError)) throw cause; return cliError(transportError(endpoint.value, cause)); } @@ -449,7 +454,11 @@ const buildEnvironmentCommand = async ( capture = read.value; } if (signal.aborted) - return cliError(new AnalysisCancelledError(CLI_COMMANDS.environmentBuild)); + return cliError( + new AnalysisCancelledError(CLI_COMMANDS.environmentBuild, { + cause: signal.reason, + }), + ); const captureInput: EnvironmentBuildCaptureInput = kind.value; const built = await buildEnvironment({ capture, diff --git a/src/gdb/GdbSessionManager.ts b/src/gdb/GdbSessionManager.ts index fc7d71f1e4..9f5e72143e 100644 --- a/src/gdb/GdbSessionManager.ts +++ b/src/gdb/GdbSessionManager.ts @@ -696,7 +696,11 @@ export class GdbSessionManager { > > { if (signal?.aborted) - return err(new AnalysisCancelledError("start_gdb_session")); + return err( + new AnalysisCancelledError("start_gdb_session", { + cause: signal.reason, + }), + ); const occupied = this.#sessions.size + this.#sessionReservations; if (occupied >= this.#maxSessions) return err( @@ -896,7 +900,11 @@ export class GdbSessionManager { session.startupResolve = resolve; session.startupReject = reject; startupAbortListener = () => - reject(new AnalysisCancelledError("start_gdb_session")); + reject( + new AnalysisCancelledError("start_gdb_session", { + cause: signal?.reason, + }), + ); signal?.addEventListener("abort", startupAbortListener, { once: true, }); @@ -923,7 +931,9 @@ export class GdbSessionManager { startupTimer.unref(); }); if (signal?.aborted) - throw new AnalysisCancelledError("start_gdb_session"); + throw new AnalysisCancelledError("start_gdb_session", { + cause: signal.reason, + }); const autoLoad = await this.#request( session, "-gdb-set auto-load off", @@ -977,11 +987,15 @@ export class GdbSessionManager { session.inferiorTtyVerified = true; } if (signal?.aborted) - throw new AnalysisCancelledError("start_gdb_session"); + throw new AnalysisCancelledError("start_gdb_session", { + cause: signal.reason, + }); // Give queued exit notifications a turn before publishing startup. await new Promise((resolve) => setImmediate(resolve)); if (signal?.aborted) - throw new AnalysisCancelledError("start_gdb_session"); + throw new AnalysisCancelledError("start_gdb_session", { + cause: signal.reason, + }); if (session.exited || session.protocolError !== undefined) throw new Error( session.protocolError ?? "GDB exited during startup readiness.", diff --git a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts index 01f4c973aa..b10f21b6d0 100644 --- a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts +++ b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts @@ -1970,19 +1970,23 @@ it("handles a start signal that becomes aborted during listener registration", a it("handles cancellation after the automatic-loading response", async () => { const controller = new AbortController(); + const reason = { source: "caller after automatic loading" }; const fake = createFakeSpawner((child, token, command) => { child.stdout.write( command.includes("list-thread-groups") ? `${token}^done,groups=[]\n` : `${token}^done\n`, ); - if (command.includes("auto-load off")) controller.abort(); + if (command.includes("auto-load off")) controller.abort(reason); }); const manager = managerWithFakeSpawner(fake.spawnProcess); const started = await manager.start(controller.signal); expect(started.ok).toBe(false); - if (!started.ok) expect(started.error._tag).toBe("AnalysisCancelledError"); + if (!started.ok) { + expect(started.error._tag).toBe("AnalysisCancelledError"); + expect(started.error.cause).toBe(reason); + } }); it("reports an MI stream failure when the child has no stdin pipe", async () => { diff --git a/tests/composition/reverseEngineering.test.ts b/tests/composition/reverseEngineering.test.ts index 7be3eac0d8..a1cf245597 100644 --- a/tests/composition/reverseEngineering.test.ts +++ b/tests/composition/reverseEngineering.test.ts @@ -12,6 +12,7 @@ import { TestReverseEngineeringService as ReverseEngineeringService } from "../s import { providerSnapshot } from "../support/reverseEngineeringCompositionFixtures.js"; import { createReverseEngineeringService } from "../../src/composition/reverseEngineering.js"; import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; +import { AnalysisCancelledError } from "../../src/domain/analysisErrorCore.js"; import { REVERSE_ENGINEERING_TOOL_CONTRACTS } from "../../src/contracts/reverseEngineeringToolContracts.js"; describe("reverse engineering command services", () => { @@ -574,6 +575,41 @@ it("shares one preflight deadline across stat and digest", async () => { expect(performance.now() - started).toBeLessThan(280); }); +it("retains an explicit null reason when digest admission is already cancelled", async () => { + const controller = new AbortController(); + controller.abort(null); + await expect( + digestFile("/tmp/rea-cancelled-digest", controller.signal), + ).rejects.toMatchObject({ + _tag: "AnalysisCancelledError", + cause: null, + }); +}); + +it("retains an object reason when cancellation interrupts an active digest stream", async () => { + const controller = new AbortController(); + const reason = { source: "caller digest cancellation" }; + const stat = async (): Promise => ({ isFile: () => true }) as Stats; + const open = () => { + const stream = new PassThrough(); + queueMicrotask(() => controller.abort(reason)); + return stream as unknown as ReturnType< + typeof import("node:fs").createReadStream + >; + }; + const failure: unknown = await digestFile( + "/tmp/rea-cancelled-stream", + controller.signal, + stat, + 1_000, + open, + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(AnalysisCancelledError); + if (!(failure instanceof AnalysisCancelledError)) + throw new Error("Expected digest cancellation"); + expect(failure.cause).toBe(reason); +}); + it("forwards selected and omitted deadlines through the production command runner", async () => { const directory = await mkdtemp(join(tmpdir(), "rea-owned-timeout-")); const path = join(directory, "fixture.bin"); From 475f09d5ca4f854cbc634074d5534717d58c6664 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:31:20 +0800 Subject: [PATCH 29/37] chore(lint): enforce buffer and cancellation regression guards --- .oxlintrc.json | 6 + docs/testing.md | 21 ++ scripts/lint/bindings.mjs | 12 ++ scripts/lint/cancellation-cause.d.mts | 4 + scripts/lint/cancellation-cause.mjs | 200 ++++++++++++++++++ scripts/lint/index.mjs | 10 + scripts/lint/no-buffer-deep-equality.d.mts | 4 + scripts/lint/no-buffer-deep-equality.mjs | 166 +++++++++++++++ src/apkSignature/ApkSignatureProvider.test.ts | 2 +- src/apkSignature/apkCertificateFacts.test.ts | 11 +- src/apktool/ApktoolProvider.test.ts | 2 +- .../environment/environmentBuild.test.ts | 10 +- .../environmentBundleWrite.test.ts | 14 +- .../environment/environmentExport.test.ts | 39 ++-- .../environmentInputSnapshot.test.ts | 9 +- .../MachOSliceArtifactReader.test.ts | 2 +- src/artifacts/NodeSeaArtifactReader.test.ts | 7 +- src/artifacts/ZipArtifactReader.test.ts | 4 +- .../apple/KeyedArchiveReader.test.ts | 21 +- src/artifacts/apple/PlistScalars.test.ts | 15 +- .../PyInstallerArtifactReader.test.ts | 49 +++-- .../pyinstaller/PyInstallerBoundaries.test.ts | 4 +- .../PyInstallerFormatBoundaries.test.ts | 15 +- src/artifacts/pyinstaller/PyzArchive.test.ts | 2 +- src/browser/BrowserScenario.test.ts | 19 +- .../protobuf/ProtobufBodyTrailer.test.ts | 6 +- .../network/PlaywrightScenarioNetwork.test.ts | 17 +- src/domain/apple/plistValue.test.ts | 2 +- .../nativeAotPe.discoveryAccounting.test.ts | 6 +- .../native/nativeAotPe.sectionRanges.test.ts | 4 +- src/domain/native/nativeAotPe.test.ts | 6 +- src/dotnet/ManagedMetadataHeaps.test.ts | 9 +- src/dotnet/ManagedPeFixtureLayout.test.ts | 28 ++- src/electron/ElectronFuseArchitecture.test.ts | 2 +- src/flutter/DartElf.test.ts | 4 +- src/ghidra/GhidraLaunchFailure.test.ts | 9 +- .../extensions/GhidraExtensions.test.ts | 9 +- src/gpu/GpuPayloadCodec.test.ts | 9 +- src/hopper/HopperImageCopy.test.ts | 4 +- src/hopper/HopperMachOImage.test.ts | 34 +-- src/ida/IdaSessionClient.test.ts | 6 +- src/native/parsers/plist.test.ts | 2 +- .../pwndbg/CoreFileMappingAuthority.test.ts | 2 +- src/process/ExecFileOutput.test.ts | 6 +- .../ReferenceSourceReaderFile.test.ts | 4 +- src/wasm/WabtArtifactProvider.test.ts | 15 +- src/wasm/WabtString.test.ts | 4 +- tests/acceptance/apkZip64FramingCli.test.ts | 4 +- .../appleBundleTraversalCoverage.test.ts | 2 +- .../applications/binaryPlistDates.test.ts | 19 +- .../hopperPublisherRejectionCli.test.ts | 2 +- .../interfaceBuilderUIKitKeyedArchive.test.ts | 2 +- .../javascriptSourceSelectionCli.test.ts | 4 +- .../applications/jvmAdmissionIdentity.test.ts | 4 +- .../keyedArchiveScalarKinds.test.ts | 8 +- .../keyedArchiveXmlNumbers.test.ts | 4 +- .../keyedArchiveXmlScalars.test.ts | 2 +- .../mobileDirectoryComponents.test.ts | 4 +- .../nativePlistBinaryScalars.test.ts | 4 +- .../applications/nativePlistNesting.test.ts | 2 +- .../applications/nibFieldIdentity.test.ts | 2 +- .../applications/packageUrlUnicodeCli.test.ts | 2 +- .../applications/peResources.test.ts | 8 +- .../resourceOnlyPlistSelection.test.ts | 8 +- .../artifactExtractionCoverage.test.ts | 8 +- tests/acceptance/devicetreeCli.test.ts | 2 +- tests/acceptance/elfNotesCli.test.ts | 6 +- tests/acceptance/erlangBeamCli.test.ts | 11 +- tests/acceptance/goBinaryCli.test.ts | 7 +- .../goBinaryPointerMappingsCli.test.ts | 2 +- tests/acceptance/godotPck.test.ts | 2 +- .../acceptance/harmonyArtifactJourney.test.ts | 2 +- tests/acceptance/il2cppMetadataCli.test.ts | 10 +- tests/acceptance/managedMetadataDelta.test.ts | 2 +- tests/acceptance/managedSelectedScope.test.ts | 4 +- tests/acceptance/modelArtifactCli.test.ts | 4 +- .../acceptance/nativeDispatchSnapshot.test.ts | 8 +- tests/acceptance/nodeSeaCli.test.ts | 4 +- tests/acceptance/pyinstallerArchive.test.ts | 63 ++++-- tests/acceptance/rustBinaryCli.test.ts | 2 +- tests/acceptance/rustUnnamedSections.test.ts | 2 +- .../clientConfigurationDiagnosticsCli.test.ts | 2 +- .../setup/geminiWorkspaceTrust.test.ts | 28 ++- .../setup/selectedSkillSetup.test.ts | 27 ++- .../setup/unqualifiedDoctorSkill.test.ts | 10 +- tests/acceptance/sqliteDatabaseCli.test.ts | 10 +- .../tools/nativeFunctionMatching.test.ts | 2 +- .../tools/protobufDescriptorFiles.test.ts | 8 +- .../tools/protobufDescriptorIdentity.test.ts | 8 +- .../tools/protobufRegistryIdentity.test.ts | 8 +- .../RecordedCrashStructureBindings.test.ts | 4 +- .../recordedCrashProvider.test.ts | 16 +- tests/boundary/browser/sazWebCapture.test.ts | 10 +- .../cli/artifactCallerSelection.test.ts | 2 +- .../cli/commandEvidenceDelivery.test.ts | 2 +- .../boundary/cli/environmentBuildCli.test.ts | 10 +- .../boundary/cli/environmentExportCli.test.ts | 30 ++- .../cli/environmentExportInputsCli.test.ts | 9 +- .../cli/idaDoctorRegistration.test.ts | 4 +- .../cli/il2cppSharedSuffixNames.test.ts | 2 +- .../cli/keyedArchivePrototypeKeyCli.test.ts | 2 +- .../cli/managedMemberSummaryCli.test.ts | 2 +- tests/boundary/cli/nativePlistCli.test.ts | 2 +- .../boundary/cli/referenceSourceUtf8.test.ts | 7 +- .../evm/evmoleInterfaceProvider.test.ts | 2 +- .../appleDispatchMetadataStrings.test.ts | 4 +- .../artifactLiteralBackslash.test.ts | 4 +- .../filesystem/artifactZipIntegrity.test.ts | 2 +- .../filesystem/asarHeaderAdmission.test.ts | 2 +- .../filesystem/asarListingAdmission.test.ts | 18 +- .../boundaryLintConfiguration.test.ts | 82 +++++++ .../filesystem/boundedFileBytes.test.ts | 5 +- .../filesystem/bufferDeepEqualityLint.test.ts | 137 ++++++++++++ .../filesystem/cancellationCauseLint.test.ts | 135 ++++++++++++ .../filesystem/ciPortableBuildCache.test.ts | 4 +- .../claudeDesktopClientConfiguration.test.ts | 2 +- .../filesystem/clientConfigurationBom.test.ts | 29 ++- .../clientConfigurationTransactions.test.ts | 4 +- .../boundary/filesystem/dosComTarget.test.ts | 4 +- .../filesystem/fileHandleChunks.test.ts | 11 +- .../ghidraProfileArtifactOwnership.test.ts | 2 +- .../keyedArchiveNumberMetadata.test.ts | 2 +- .../keyedArchiveScalarIdentity.test.ts | 2 +- .../filesystem/rawBinaryTarget.test.ts | 2 +- .../regularFileCancellation.test.ts | 2 +- .../safeOutputTreePublication.test.ts | 9 +- .../setupClientConfigCleanupOwnership.test.ts | 6 +- tests/boundary/filesystem/setupSkill.test.ts | 4 +- .../filesystem/setupSkillPublication.test.ts | 12 +- .../filesystem/stableArtifact.test.ts | 9 +- .../filesystem/staticCheckReports.test.ts | 7 +- .../filesystem/targetDiagnostics.test.ts | 2 +- .../filesystem/verifierCommandOutput.test.ts | 19 +- .../filesystem/verifierFileSnapshots.test.ts | 4 +- .../filesystem/webScriptExport.test.ts | 60 ++++-- .../firmware/firmwarePublicationPaths.test.ts | 2 +- .../firmware/firmwareReportFiles.test.ts | 2 +- .../firmwareWorkspaceRecovery.test.ts | 2 +- tests/boundary/gpu/codeObjects.test.ts | 4 +- .../boundary/il2cpp/il2cppBinaryFile.test.ts | 4 +- .../il2cpp/il2cppMappingOwnership.test.ts | 10 +- .../mcp/asarListingAdmissionMcp.test.ts | 2 +- .../mcp/asarUnpackedExtractionMcp.test.ts | 4 +- tests/boundary/mcp/elfNotesMcp.test.ts | 4 +- .../boundary/mcp/environmentBundleMcp.test.ts | 17 +- tests/boundary/mcp/ghidraReopen.test.ts | 2 +- tests/boundary/mcp/goBinaryViews.test.ts | 2 +- tests/boundary/mcp/peLoadConfigMcp.test.ts | 2 +- .../mcp/selectedSnapshotCleanupMcp.test.ts | 4 +- tests/boundary/mcp/sessionMcp.test.ts | 2 +- tests/boundary/mcp/wasmArtifactMcp.test.ts | 9 +- .../mcp/webScriptSelectionMcp.test.ts | 9 +- tests/boundary/pcap/pcapProvider.test.ts | 15 +- .../process/modelFixtureShutdown.test.ts | 4 +- .../nativeVerifierCommandOwnership.test.ts | 2 +- .../nativeVerifierPreflightOwnership.test.ts | 4 +- .../process/packagePackIsolation.test.ts | 2 +- .../process/packedBridgeVerification.test.ts | 4 +- .../process/verifierCommandSemantics.test.ts | 10 +- .../adb/adbHashCleanupConsumers.test.ts | 9 +- .../adb/adbInputSnapshotOwnership.test.ts | 22 +- .../adbScreenPublicationCancellation.test.ts | 24 ++- .../ghidra/ghidraTargetSnapshot.test.ts | 4 +- .../providers/hopper/bridgeLauncher.test.ts | 2 +- ...actProvider.extractionCompleteness.test.ts | 18 +- .../artifactProvider.machOSlices.test.ts | 2 +- .../artifactProvider.zipCancellation.test.ts | 9 +- .../native/nativePlistTextSource.test.ts | 2 +- .../nativeSignatureTargetBinding.test.ts | 19 +- .../linuxHopperPublisherVerification.test.ts | 4 +- .../signals/recordingProvider.test.ts | 2 +- .../sqlite/sqliteDatabaseInspection.test.ts | 2 +- .../sqlite/sqliteDatabaseProvider.test.ts | 22 +- .../sqlite/sqliteDatabaseSnapshot.test.ts | 10 +- tests/fixtures/machoArmSlices.ts | 4 +- .../artifactFileSourceLifecycle.test.ts | 2 +- .../asarSnapshotOperation.test.ts | 4 +- .../cutterDescriptorLifetime.test.ts | 7 +- .../ghidraImportSnapshotLifetime.test.ts | 2 +- .../mainStartupCancellation.test.ts | 4 +- .../regularFileNoFollowFallback.test.ts | 4 +- .../setupConfigurationConflicts.test.ts | 2 +- 182 files changed, 1653 insertions(+), 497 deletions(-) create mode 100644 scripts/lint/bindings.mjs create mode 100644 scripts/lint/cancellation-cause.d.mts create mode 100644 scripts/lint/cancellation-cause.mjs create mode 100644 scripts/lint/index.mjs create mode 100644 scripts/lint/no-buffer-deep-equality.d.mts create mode 100644 scripts/lint/no-buffer-deep-equality.mjs create mode 100644 tests/boundary/filesystem/boundaryLintConfiguration.test.ts create mode 100644 tests/boundary/filesystem/bufferDeepEqualityLint.test.ts create mode 100644 tests/boundary/filesystem/cancellationCauseLint.test.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index a4618ad114..0a760d3d6b 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -1,7 +1,13 @@ { "$schema": "./node_modules/oxlint/configuration_schema.json", "plugins": ["typescript", "import", "unicorn", "oxc"], + "jsPlugins": [{ "name": "rea", "specifier": "./scripts/lint/index.mjs" }], "rules": { + "rea/no-buffer-deep-equality": "error", + "rea/cancellation-cause": "error", + "no-async-promise-executor": "error", + "no-unreachable": "error", + "no-unsafe-optional-chaining": "error", "complexity": ["warn", 20], "max-depth": ["warn", 4], "max-lines": [ diff --git a/docs/testing.md b/docs/testing.md index b7b8f0750d..7af365f6f2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -146,6 +146,27 @@ boundaries and architecture guards enforce their declared import and source rules. Passing them does not prove that two parsers agree, a wrapper is still needed at runtime, or cleanup has one owner. +The lint gate treats `no-async-promise-executor`, `no-unreachable`, and +`no-unsafe-optional-chaining` as errors. These catch lost async executor failures, +dead code, and unsafe uses of optional-chain results. `no-unsafe-finally` remains +a warning because cleanup paths can deliberately throw aggregated failures; +review whether they preserve the primary failure. Two local rules +cover narrower observed failures: `rea/no-buffer-deep-equality` flags Vitest +deep comparisons of Buffers from recognized constructors and file reads, +including simple `const` aliases; `rea/cancellation-cause` requires an explicit +cause when a directly checked caller `AbortSignal` is converted to an +`AnalysisCancelledError`. The rules do not infer arbitrary Buffer types, +follow indirect cancellation helpers, or prove that an explicit cause equals +the caller's reason. Keep focused boundary and lifecycle tests for those facts. + +Type-aware linting remains outside the routine gate: a full source, test, and +script scan exceeded the bounded process-family memory envelope, including +with Go runtime limits. Future scoped evaluation can consider +`typescript/no-floating-promises`, `typescript/return-await` with +`error-handling-correctness-only`, `typescript/no-misused-promises`, and +`typescript/switch-exhaustiveness-check`. None replaces behavioral checks for +source identity, omission versus unknown, or Evidence provenance. + Routine iterations and rebases need focused regressions and relevant checks. Before handing off a PR, run `npm run check` and generated-document checks when applicable; CI owns the full suite and coverage. Use the full local gate for diff --git a/scripts/lint/bindings.mjs b/scripts/lint/bindings.mjs new file mode 100644 index 0000000000..400304c6a0 --- /dev/null +++ b/scripts/lint/bindings.mjs @@ -0,0 +1,12 @@ +/** Resolve the lexical binding instead of treating a matching name as an API. */ +export const findBinding = (sourceCode, identifier) => { + for ( + let scope = sourceCode.getScope(identifier); + scope !== null; + scope = scope.upper + ) { + const variable = scope.set.get(identifier.name); + if (variable !== undefined) return variable; + } + return undefined; +}; diff --git a/scripts/lint/cancellation-cause.d.mts b/scripts/lint/cancellation-cause.d.mts new file mode 100644 index 0000000000..faf40667bb --- /dev/null +++ b/scripts/lint/cancellation-cause.d.mts @@ -0,0 +1,4 @@ +import type { RuleTester } from "oxlint/plugins-dev"; + +declare const cancellationCauseRule: Parameters[1]; +export default cancellationCauseRule; diff --git a/scripts/lint/cancellation-cause.mjs b/scripts/lint/cancellation-cause.mjs new file mode 100644 index 0000000000..050c76df0b --- /dev/null +++ b/scripts/lint/cancellation-cause.mjs @@ -0,0 +1,200 @@ +import { findBinding } from "./bindings.mjs"; + +/** + * Require an explicit cancellation reason when a directly checked caller signal + * is converted to an AnalysisCancelledError. This is deliberately syntactic: + * internal shutdown and cancellation forwarded through other owners are outside + * the rule. + */ + +const coreModule = /(?:^|\/)analysisErrorCore(?:\.js|\.ts)?$/; + +const unwrapChain = (node) => + node?.type === "ChainExpression" ? node.expression : node; + +const propertyName = (property) => + property.computed + ? property.property.type === "Literal" + ? property.property.value + : undefined + : property.property.name; + +const annotationType = (node) => { + const annotation = node?.typeAnnotation?.typeAnnotation; + return annotation?.type === "TSTypeReference" && + annotation.typeName.type === "Identifier" + ? annotation.typeName + : undefined; +}; + +const soleDefinition = (variable) => + variable?.defs.length === 1 ? variable.defs[0] : undefined; + +const isKnownType = (node, kind, context) => { + const type = annotationType(node); + if (type === undefined) return false; + const definition = soleDefinition(findBinding(context.sourceCode, type)); + if (kind === "AbortSignal") + return type.name === "AbortSignal" && definition === undefined; + return ( + definition?.type === "ImportBinding" && + definition.parent?.type === "ImportDeclaration" && + /(?:^|\/)AnalysisProvider(?:\.js|\.ts)?$/.test( + definition.parent.source.value, + ) && + definition.node.type === "ImportSpecifier" && + definition.node.imported.name === "ExecutionOptions" + ); +}; + +const unmodified = (variable) => + variable !== undefined && + variable.references.every( + (reference) => !reference.isWrite() || reference.init, + ); + +const isOptionsSignal = (node, context, seen) => { + node = unwrapChain(node); + if (node?.type !== "MemberExpression" || propertyName(node) !== "signal") + return false; + return isKnownBinding(node.object, "ExecutionOptions", context, seen); +}; + +const isKnownBinding = (node, kind, context, seen) => { + if (node?.type !== "Identifier") return false; + const variable = findBinding(context.sourceCode, node); + if (!unmodified(variable) || seen.has(variable)) return false; + seen.add(variable); + const definition = soleDefinition(variable); + if (definition?.type === "Parameter") + return isKnownType(definition.name, kind, context); + if (definition?.type !== "Variable" || definition.parent?.kind !== "const") + return false; + const declaration = definition.node; + if (isKnownType(declaration.id, kind, context)) return true; + return kind === "AbortSignal" + ? isKnownSignal(declaration.init, context, seen) + : false; +}; + +const isKnownSignal = (node, context, seen = new Set()) => + isKnownBinding(unwrapChain(node), "AbortSignal", context, seen) || + isOptionsSignal(node, context, seen); + +const isAbortRead = (node, context) => { + node = unwrapChain(node); + return ( + node?.type === "MemberExpression" && + propertyName(node) === "aborted" && + isKnownSignal(node.object, context) + ); +}; + +const isTrueLiteral = (node) => node?.type === "Literal" && node.value === true; + +const positiveAbortGuard = (test, context) => { + if (isAbortRead(test, context)) return true; + if ( + test?.type === "BinaryExpression" && + (test.operator === "===" || test.operator === "==") + ) + return ( + (isAbortRead(test.left, context) && isTrueLiteral(test.right)) || + (isTrueLiteral(test.left) && isAbortRead(test.right, context)) + ); + if (test?.type === "LogicalExpression" && test.operator === "&&") + return ( + positiveAbortGuard(test.left, context) || + positiveAbortGuard(test.right, context) + ); + return false; +}; + +const isCanonicalConstructor = (node, context) => { + if (node?.type !== "Identifier") return false; + const definition = soleDefinition(findBinding(context.sourceCode, node)); + return ( + definition?.type === "ImportBinding" && + definition.parent?.type === "ImportDeclaration" && + coreModule.test(definition.parent.source.value) && + definition.node.type === "ImportSpecifier" && + definition.node.imported.name === "AnalysisCancelledError" + ); +}; + +const hasExplicitCause = (node) => { + const options = node.arguments[1]; + if (options !== undefined && options.type !== "ObjectExpression") return true; // Forwarded context is owned elsewhere. + if (options === undefined) return false; + return options.properties.some( + (property) => + property.type === "SpreadElement" || + (property.type === "Property" && + (property.computed + ? property.key.type !== "Literal" || property.key.value === "cause" + : (property.key.type === "Identifier" + ? property.key.name + : property.key.value) === "cause")), + ); +}; + +const inspectExpression = (expression, context) => { + if (expression?.type === "NewExpression") { + if ( + isCanonicalConstructor(expression.callee, context) && + !hasExplicitCause(expression) + ) + context.report({ + node: expression, + messageId: "missingCause", + }); + return; + } + if (expression?.type === "CallExpression") { + // Common Result adapter: return err(new AnalysisCancelledError(...)). + for (const argument of expression.arguments) + inspectExpression(argument, context); + } else if (expression?.type === "AssignmentExpression") { + inspectExpression(expression.right, context); + } +}; + +const inspectBranch = (branch, context) => { + if (branch.type === "BlockStatement") { + for (const statement of branch.body) inspectBranch(statement, context); + } else if (branch.type === "VariableDeclaration") { + for (const declaration of branch.declarations) + inspectExpression(declaration.init, context); + } else if ( + branch.type === "ThrowStatement" || + branch.type === "ReturnStatement" + ) { + inspectExpression(branch.argument, context); + } else if (branch.type === "ExpressionStatement") { + inspectExpression(branch.expression, context); + } +}; + +const cancellationCauseRule = { + meta: { + type: "problem", + docs: { + description: + "Preserve a directly checked caller AbortSignal reason in AnalysisCancelledError", + }, + messages: { + missingCause: + "Pass the checked AbortSignal's reason as an explicit cancellation cause.", + }, + }, + create(context) { + return { + IfStatement(node) { + if (positiveAbortGuard(node.test, context)) + inspectBranch(node.consequent, context); + }, + }; + }, +}; + +export default cancellationCauseRule; diff --git a/scripts/lint/index.mjs b/scripts/lint/index.mjs new file mode 100644 index 0000000000..c04cbd95c4 --- /dev/null +++ b/scripts/lint/index.mjs @@ -0,0 +1,10 @@ +import noBufferDeepEquality from "./no-buffer-deep-equality.mjs"; +import cancellationCause from "./cancellation-cause.mjs"; + +export default { + meta: { name: "rea" }, + rules: { + "no-buffer-deep-equality": noBufferDeepEquality, + "cancellation-cause": cancellationCause, + }, +}; diff --git a/scripts/lint/no-buffer-deep-equality.d.mts b/scripts/lint/no-buffer-deep-equality.d.mts new file mode 100644 index 0000000000..08eb75e22b --- /dev/null +++ b/scripts/lint/no-buffer-deep-equality.d.mts @@ -0,0 +1,4 @@ +import type { RuleTester } from "oxlint/plugins-dev"; + +declare const noBufferDeepEquality: Parameters[1]; +export default noBufferDeepEquality; diff --git a/scripts/lint/no-buffer-deep-equality.mjs b/scripts/lint/no-buffer-deep-equality.mjs new file mode 100644 index 0000000000..8aaff86bb3 --- /dev/null +++ b/scripts/lint/no-buffer-deep-equality.mjs @@ -0,0 +1,166 @@ +import { findBinding } from "./bindings.mjs"; + +const memberName = (node) => + node?.type === "MemberExpression" && + !node.computed && + node.property.type === "Identifier" + ? node.property.name + : null; + +const unwrap = (node) => { + while ( + node?.type === "TSAsExpression" || + node?.type === "TSSatisfiesExpression" || + node?.type === "TSNonNullExpression" || + node?.type === "TSTypeAssertion" || + node?.type === "ParenthesizedExpression" + ) + node = node.expression; + return node; +}; + +const importedKind = (variable) => { + const definition = variable?.defs[0]; + if (definition?.type !== "ImportBinding") return null; + const source = definition.parent?.source?.value; + const node = definition.node; + if (node?.type === "ImportNamespaceSpecifier") { + if (source === "node:fs/promises") return "fsPromises"; + if (source === "node:fs") return "fs"; + return null; + } + if (node?.type !== "ImportSpecifier") return null; + const imported = node.imported?.name ?? node.imported?.value; + if (source === "vitest" && imported === "expect") return "expect"; + if (source === "node:buffer" && imported === "Buffer") return "Buffer"; + if (source === "node:fs/promises" && imported === "readFile") + return "readFile"; + if (source === "node:fs" && imported === "readFileSync") + return "readFileSync"; + return null; +}; + +const constInitializer = (variable) => { + if (variable?.defs.length !== 1) return null; + const definition = variable.defs[0]; + if ( + definition?.type !== "Variable" || + definition.node?.type !== "VariableDeclarator" || + definition.parent?.kind !== "const" || + variable.references.some( + (reference) => reference.isWrite() && !reference.init, + ) + ) + return null; + return definition.node.init; +}; + +/** Flag byte comparisons that make Vitest enumerate a Node Buffer. */ +const noBufferDeepEquality = { + meta: { + type: "suggestion", + messages: { + nativeBytes: + "Compare Node Buffers with Buffer.equals or Buffer.compare; deep equality can enumerate every byte and exhaust memory.", + }, + }, + create(context) { + const { sourceCode } = context; + const symbolKind = (expression, seen = new Set()) => { + const node = unwrap(expression); + if (node?.type === "Identifier") { + const variable = findBinding(sourceCode, node); + if ( + (variable === undefined || variable.defs.length === 0) && + sourceCode.isGlobalReference(node) + ) + return node.name === "Buffer" ? "Buffer" : null; + const imported = importedKind(variable); + if (imported !== null) return imported; + if (variable === undefined || seen.has(variable)) return null; + const initializer = constInitializer(variable); + if (initializer === null) return null; + seen.add(variable); + return symbolKind(initializer, seen); + } + const property = memberName(node); + if (property === null) return null; + const owner = symbolKind(node.object, seen); + if (owner === "fsPromises" && property === "readFile") return "readFile"; + if (owner === "fs" && property === "readFileSync") return "readFileSync"; + if ( + owner === "Buffer" && + ["from", "alloc", "allocUnsafe", "allocUnsafeSlow", "concat"].includes( + property, + ) + ) + return "bufferFactory"; + return null; + }; + const isBuffer = (expression, seen = new Set(), awaited = false) => { + const node = unwrap(expression); + if (node?.type === "AwaitExpression") + return isBuffer(node.argument, seen, true); + if (node?.type === "Identifier") { + const variable = findBinding(sourceCode, node); + if (variable === undefined || seen.has(variable)) return false; + const initializer = constInitializer(variable); + if (initializer === null) return false; + seen.add(variable); + return isBuffer(initializer, seen, awaited); + } + if (node?.type !== "CallExpression" || node.optional) return false; + const kind = symbolKind(node.callee); + return ( + kind === "bufferFactory" || + ((kind === "readFileSync" || (kind === "readFile" && awaited)) && + node.arguments.length === 1 && + node.arguments[0].type !== "SpreadElement") + ); + }; + const expectedCall = (object, settlement = null) => { + const node = unwrap(object); + const property = memberName(node); + if (property === "not") return expectedCall(node.object, settlement); + if (property === "resolves" || property === "rejects") + return expectedCall(node.object, property); + return node?.type === "CallExpression" && + !node.optional && + symbolKind(node.callee) === "expect" + ? { call: node, settlement } + : null; + }; + return { + CallExpression(node) { + if (node.optional) return; + const matcher = memberName(node.callee); + if (matcher !== "toEqual" && matcher !== "toStrictEqual") return; + const assertion = expectedCall(node.callee.object); + if ( + assertion === null || + ![1, 2].includes(assertion.call.arguments.length) || + assertion.call.arguments.some( + (argument) => argument.type === "SpreadElement", + ) || + node.arguments.length !== 1 + ) + return; + if ( + (assertion.settlement !== "rejects" && + isBuffer( + assertion.call.arguments[0], + new Set(), + assertion.settlement === "resolves", + )) || + isBuffer(node.arguments[0]) + ) + context.report({ + node: node.callee.property, + messageId: "nativeBytes", + }); + }, + }; + }, +}; + +export default noBufferDeepEquality; diff --git a/src/apkSignature/ApkSignatureProvider.test.ts b/src/apkSignature/ApkSignatureProvider.test.ts index c9619fc471..f8568b38a4 100644 --- a/src/apkSignature/ApkSignatureProvider.test.ts +++ b/src/apkSignature/ApkSignatureProvider.test.ts @@ -524,7 +524,7 @@ const registerSignatureInputAdmissionCases = () => { corrupt.writeUInt32LE(0x06064b51, endOffset); const restored = Buffer.from(corrupt); restored.writeUInt32LE(0x06064b50, endOffset); - expect(restored).toEqual(valid); + expect(Buffer.compare(restored, valid)).toBe(0); for (const [name, bytes] of [ ["valid", valid], ["restored", restored], diff --git a/src/apkSignature/apkCertificateFacts.test.ts b/src/apkSignature/apkCertificateFacts.test.ts index b5d2adfa9d..8764ed4c84 100644 --- a/src/apkSignature/apkCertificateFacts.test.ts +++ b/src/apkSignature/apkCertificateFacts.test.ts @@ -18,10 +18,13 @@ it("binds readable certificate fields to the exact carried DER despite embedded createHash("sha256").update(outer).digest("hex"), ); expect( - new X509Certificate( - `-----BEGIN CERTIFICATE-----\n${Buffer.from(outer).toString("base64")}\n-----END CERTIFICATE-----\n`, - ).raw, - ).toEqual(Buffer.from(outer)); + Buffer.compare( + new X509Certificate( + `-----BEGIN CERTIFICATE-----\n${Buffer.from(outer).toString("base64")}\n-----END CERTIFICATE-----\n`, + ).raw, + Buffer.from(outer), + ), + ).toBe(0); } }); it.each([ diff --git a/src/apktool/ApktoolProvider.test.ts b/src/apktool/ApktoolProvider.test.ts index 707bd933ed..0c1b1e8424 100644 --- a/src/apktool/ApktoolProvider.test.ts +++ b/src/apktool/ApktoolProvider.test.ts @@ -250,7 +250,7 @@ sdkInfo: coverage: "complete", }, }); - expect(await readFile(apk.path)).toEqual(original); + expect(Buffer.compare(await readFile(apk.path), original)).toBe(0); }); it.each([ diff --git a/src/application/environment/environmentBuild.test.ts b/src/application/environment/environmentBuild.test.ts index 275071433f..f62f5f5302 100644 --- a/src/application/environment/environmentBuild.test.ts +++ b/src/application/environment/environmentBuild.test.ts @@ -150,9 +150,13 @@ const sameTree = async (left: string, right: string): Promise => { const files = await listFiles(left); expect(await listFiles(right)).toEqual(files); for (const file of files) - expect(await readFile(join(left, file)), file).toEqual( - await readFile(join(right, file)), - ); + expect( + Buffer.compare( + await readFile(join(left, file)), + await readFile(join(right, file)), + ), + file, + ).toBe(0); return files.length; }; diff --git a/src/application/environment/environmentBundleWrite.test.ts b/src/application/environment/environmentBundleWrite.test.ts index 8422eb3de8..592316fcea 100644 --- a/src/application/environment/environmentBundleWrite.test.ts +++ b/src/application/environment/environmentBundleWrite.test.ts @@ -66,8 +66,11 @@ it("retains the admitted tarball descriptor and refuses a pathname replacement d error: { _tag: "AnalysisArtifactChangedError", path: tarball }, }); expect( - await readFile(join(directory, "envs/grafana-example/rea-agents.tgz")), - ).toEqual(originalBytes); + Buffer.compare( + await readFile(join(directory, "envs/grafana-example/rea-agents.tgz")), + originalBytes, + ), + ).toBe(0); } finally { writer.mockRestore(); } @@ -158,8 +161,11 @@ it("uses confirmed publication after the output owner's descriptor cleanup compl }); expect(result).toMatchObject({ ok: true, value: { status: "exported" } }); expect( - await readFile(join(directory, "envs/grafana-example/evidence.json")), - ).toEqual(inputs.value.evidence.bytes); + Buffer.compare( + await readFile(join(directory, "envs/grafana-example/evidence.json")), + inputs.value.evidence.bytes, + ), + ).toBe(0); } finally { commit.mockRestore(); } diff --git a/src/application/environment/environmentExport.test.ts b/src/application/environment/environmentExport.test.ts index d14c72b3d2..1691944ab5 100644 --- a/src/application/environment/environmentExport.test.ts +++ b/src/application/environment/environmentExport.test.ts @@ -62,13 +62,19 @@ describe("readEnvironmentExportInputs", () => { ); if (!inputs.ok) throw inputs.error; expect(inputs.value.manifest.path).toBe(files.manifestPath); - expect(inputs.value.manifest.bytes).toEqual( - await readFile(files.manifestPath), - ); + expect( + Buffer.compare( + inputs.value.manifest.bytes, + await readFile(files.manifestPath), + ), + ).toBe(0); expect(inputs.value.manifest.value).toEqual(fixture.manifest); - expect(inputs.value.evidence.bytes).toEqual( - await readFile(files.capturePath), - ); + expect( + Buffer.compare( + inputs.value.evidence.bytes, + await readFile(files.capturePath), + ), + ).toBe(0); expect(inputs.value.evidence.value).toEqual(fixture.capture); }); @@ -151,11 +157,17 @@ const registerBundlePublicationCases = () => { (await readFile(join(directory, file.path))).length, ); expect( - await readFile(join(directory, "envs/grafana-example/manifest.json")), - ).toEqual(await readFile(files.manifestPath)); + Buffer.compare( + await readFile(join(directory, "envs/grafana-example/manifest.json")), + await readFile(files.manifestPath), + ), + ).toBe(0); expect( - await readFile(join(directory, "envs/grafana-example/evidence.json")), - ).toEqual(input.evidence.bytes); + Buffer.compare( + await readFile(join(directory, "envs/grafana-example/evidence.json")), + input.evidence.bytes, + ), + ).toBe(0); expect(await listFiles(root)).toEqual( expect.arrayContaining(["capture.json", "manifest.json"]), ); @@ -183,8 +195,11 @@ const registerBundlePublicationCases = () => { bytes: 7, }); expect( - await readFile(join(directory, "envs/replay-copy/rea-agents.tgz")), - ).toEqual(await readFile(tarball)); + Buffer.compare( + await readFile(join(directory, "envs/replay-copy/rea-agents.tgz")), + await readFile(tarball), + ), + ).toBe(0); expect( await readFile(join(directory, "envs/replay-copy/Dockerfile"), "utf8"), ).toContain("COPY rea-agents.tgz /tmp/"); diff --git a/src/application/environment/environmentInputSnapshot.test.ts b/src/application/environment/environmentInputSnapshot.test.ts index cfb3510077..6739e6e73e 100644 --- a/src/application/environment/environmentInputSnapshot.test.ts +++ b/src/application/environment/environmentInputSnapshot.test.ts @@ -43,9 +43,12 @@ it("seals privately parsed files and shares reported trees across both Evidence expect(webNetworkCaptureSchema.parse(normalized).records[0]?.reported).toBe( reported, ); - expect(inputs.value.evidence.bytes).toEqual( - await readFile(files.capturePath), - ); + expect( + Buffer.compare( + inputs.value.evidence.bytes, + await readFile(files.capturePath), + ), + ).toBe(0); expect( immutableJsonSnapshotMember( inputs.value.evidence.value, diff --git a/src/artifacts/MachOSliceArtifactReader.test.ts b/src/artifacts/MachOSliceArtifactReader.test.ts index 85c6a2537f..be5d1216a0 100644 --- a/src/artifacts/MachOSliceArtifactReader.test.ts +++ b/src/artifacts/MachOSliceArtifactReader.test.ts @@ -371,7 +371,7 @@ describe("MachOSliceArtifactReader", () => { const verifiedRead = await reader.open(verifiedArmEntry); for await (const chunk of verifiedRead) verifiedChunks.push(Buffer.from(chunk)); - expect(Buffer.concat(verifiedChunks)).toEqual(arm); + expect(Buffer.compare(Buffer.concat(verifiedChunks), arm)).toBe(0); const truncatedController = new AbortController(); const truncated = await reader.open( diff --git a/src/artifacts/NodeSeaArtifactReader.test.ts b/src/artifacts/NodeSeaArtifactReader.test.ts index 56c7fceb5b..7c4690b824 100644 --- a/src/artifacts/NodeSeaArtifactReader.test.ts +++ b/src/artifacts/NodeSeaArtifactReader.test.ts @@ -408,8 +408,11 @@ it("extracts the main script and assets without executing them", async () => { await readFile(join(extracted.output_root, "assets/message.txt"), "utf8"), ).toBe("hello"); expect( - await readFile(join(extracted.output_root, "__node_sea__/code-cache")), - ).toEqual(Buffer.from("cache")); + Buffer.compare( + await readFile(join(extracted.output_root, "__node_sea__/code-cache")), + Buffer.from("cache"), + ), + ).toBe(0); const extractedInventory = await scanCanonicalArtifactInventory( extracted.output_root, ); diff --git a/src/artifacts/ZipArtifactReader.test.ts b/src/artifacts/ZipArtifactReader.test.ts index be60df9835..79da39efd1 100644 --- a/src/artifacts/ZipArtifactReader.test.ts +++ b/src/artifacts/ZipArtifactReader.test.ts @@ -103,7 +103,9 @@ describe("ZipArtifactReader positional reads", () => { const chunks: Buffer[] = []; for await (const chunk of await reader.open(entry)) chunks.push(Buffer.from(chunk)); - expect(Buffer.concat(chunks)).toEqual(Buffer.from(fixture.content)); + expect( + Buffer.compare(Buffer.concat(chunks), Buffer.from(fixture.content)), + ).toBe(0); expect(reads.mock.calls.length).toBeGreaterThan(3); } finally { reads.mockRestore(); diff --git a/src/artifacts/apple/KeyedArchiveReader.test.ts b/src/artifacts/apple/KeyedArchiveReader.test.ts index 39b4de420a..93bef7e930 100644 --- a/src/artifacts/apple/KeyedArchiveReader.test.ts +++ b/src/artifacts/apple/KeyedArchiveReader.test.ts @@ -578,7 +578,7 @@ describe("binary keyed archive dates", () => { dateScalar("2001-01-01T00:00:00.100Z"), ); expect(graph.limitations).not.toContain(unknownDateLimitation(1)); - expect(bytes).toEqual(pythonBinaryDateControl); + expect(Buffer.compare(bytes, pythonBinaryDateControl)).toBe(0); expect(createHash("sha256").update(bytes).digest("hex")).toBe( "39f08db8341bd74220352dfd559485515cce1e6035e08f27a59ab8f856374a30", ); @@ -593,7 +593,9 @@ describe("binary keyed archive dates", () => { dateScalar("2001-01-01T00:00:00.100Z"), ); expect(inspected.limitations).not.toContain(unknownDateLimitation(1)); - expect(await readFile(join(root, "archive.plist"))).toEqual(bytes); + expect( + Buffer.compare(await readFile(join(root, "archive.plist")), bytes), + ).toBe(0); }); it.each( @@ -612,7 +614,7 @@ describe("binary keyed archive dates", () => { `binary plist date marker 0x${marker.toString(16)} is invalid`, ), ); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); if (marker === 0x30) expect(createHash("sha256").update(bytes).digest("hex")).toBe( "ac5696b866c312feb53c2e82edd8fea79dbeb572eb1b07e4291603faa41e8c0e", @@ -631,7 +633,7 @@ describe("binary keyed archive dates", () => { reason: "format", message: `Cannot decode selected Foundation keyed archive: binary plist date marker 0x${marker.toString(16)} is invalid`, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); @@ -649,7 +651,7 @@ describe("binary keyed archive dates", () => { expect(graph.limitations.join("\n")).not.toContain( "number metadata could not be completely observed", ); - expect(submillisecond).toEqual(original); + expect(Buffer.compare(submillisecond, original)).toBe(0); expect(createHash("sha256").update(submillisecond).digest("hex")).toBe( "ab69ba6e8034c6f60b828ee323a5a3e81bae3e7657477e0ebfbfdb93d53ed2ce", ); @@ -662,7 +664,12 @@ describe("binary keyed archive dates", () => { }); expect(inspected.objects[1]).toMatchObject(dateScalar(null)); expect(inspected.limitations).toContain(unknownDateLimitation(1)); - expect(await readFile(join(root, "archive.plist"))).toEqual(submillisecond); + expect( + Buffer.compare( + await readFile(join(root, "archive.plist")), + submillisecond, + ), + ).toBe(0); const mixed = Buffer.from( buildBinary({ @@ -697,7 +704,7 @@ describe("binary keyed archive dates", () => { { $plist_type: "date", iso8601: null }, ]); expect(mixedGraph.limitations).toContain(unknownDateLimitation(2)); - expect(mixed).toEqual(mixedOriginal); + expect(Buffer.compare(mixed, mixedOriginal)).toBe(0); }); }); diff --git a/src/artifacts/apple/PlistScalars.test.ts b/src/artifacts/apple/PlistScalars.test.ts index 1074d8c2d6..41d9a434da 100644 --- a/src/artifacts/apple/PlistScalars.test.ts +++ b/src/artifacts/apple/PlistScalars.test.ts @@ -321,9 +321,12 @@ describe("binary plist number rewriting", () => { uidOffsetsToNormalize: [], }), ).toBe(bytes); - expect(bytes.subarray(13, 30)).toEqual( - Buffer.from([0x14, ...Array(16).fill(0xff)]), - ); + expect( + Buffer.compare( + bytes.subarray(13, 30), + Buffer.from([0x14, ...Array(16).fill(0xff)]), + ), + ).toBe(0); }); it("does not rewrite a wide integer when a date aliases its payload", () => { @@ -362,7 +365,7 @@ describe("binary plist number rewriting", () => { }, { $plist_type: "date", iso8601: null }, ]); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); expect(bytes[11]).toBe(0x14); expect(bytes[18]).toBe(0x33); }); @@ -424,7 +427,7 @@ describe("binary plist shared object-table entries", () => { expect(numbers.limitations().join("\n")).not.toContain( "could not be completely observed", ); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); }); it("rewrites a shared 16-byte integer once for every entry", () => { @@ -450,7 +453,7 @@ describe("binary plist shared object-table entries", () => { { $plist_type: "integer", decimal: "18446744073709551618" }, { $plist_type: "integer", decimal: "1152921504606846979" }, ]); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); }); it("records a shared date once and keeps its sibling exact", () => { diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index c3d759f57a..3e79b0b211 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -373,14 +373,19 @@ it("extracts derived members into the safe output tree", async () => { path, snapshot, ); - expect(await readFile(join(output, "hello"))).toEqual(derived(script)); + expect( + Buffer.compare(await readFile(join(output, "hello")), derived(script)), + ).toBe(0); expect( extracted.artifacts.find((artifact) => artifact.relative_path === "hello") ?.sha256, ).toBe(sha256(derived(script))); - expect(await readFile(join(output, "PYZ.pyz", "app.pyc"))).toEqual( - derived(Buffer.from("app-body")), - ); + expect( + Buffer.compare( + await readFile(join(output, "PYZ.pyz", "app.pyc")), + derived(Buffer.from("app-body")), + ), + ).toBe(0); }); it("reports encrypted PYZ entries and refuses to extract them", async () => { @@ -509,7 +514,9 @@ it.each([false, true])( path, snapshot, ); - expect(await readFile(join(outputRoot, "PYZ.pyz"))).toEqual(payload); + expect( + Buffer.compare(await readFile(join(outputRoot, "PYZ.pyz")), payload), + ).toBe(0); }, ); @@ -808,7 +815,9 @@ it("projects CArchive type n entries as symlinks", async () => { await expect(readFile(join(output, "libfoo.so"))).rejects.toMatchObject({ code: "ENOENT", }); - expect(await readFile(join(output, "data.bin"))).toEqual(payload); + expect( + Buffer.compare(await readFile(join(output, "data.bin")), payload), + ).toBe(0); }); it.each([false, true])( @@ -948,10 +957,15 @@ it("inventories and extracts a Windows member name under its canonical path", as path, snapshot, ); - expect(await readFile(join(output, "pkg", "data.bin"))).toEqual(payload); - expect(await readFile(join(output, "PYZ.pyz", "res", "icon.bin"))).toEqual( - icon, - ); + expect( + Buffer.compare(await readFile(join(output, "pkg", "data.bin")), payload), + ).toBe(0); + expect( + Buffer.compare( + await readFile(join(output, "PYZ.pyz", "res", "icon.bin")), + icon, + ), + ).toBe(0); }); it("preserves distinct POSIX backslash and slash member paths", async () => { @@ -1008,8 +1022,15 @@ it("preserves distinct POSIX backslash and slash member paths", async () => { path, snapshot, ); - expect(await readFile(join(output, backslashName))).toEqual(backslashPayload); - expect(await readFile(join(output, "a", "b.txt"))).toEqual(slashPayload); + expect( + Buffer.compare( + await readFile(join(output, backslashName)), + backslashPayload, + ), + ).toBe(0); + expect( + Buffer.compare(await readFile(join(output, "a", "b.txt")), slashPayload), + ).toBe(0); }); it("reads a PE signature beyond the carrier prefix before projecting paths", async () => { @@ -1304,7 +1325,9 @@ it("uses known PYZ magic for separate CArchive scripts when the marshal table st path, snapshot, ); - expect(await readFile(join(output, "hello"))).toEqual(derived(script)); + expect( + Buffer.compare(await readFile(join(output, "hello")), derived(script)), + ).toBe(0); }); it("retains colliding CArchive rows and PYZ children but refuses extraction", async () => { diff --git a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts index 808b1f8bfc..8c46ce1593 100644 --- a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts @@ -656,7 +656,9 @@ for (const cancel of [false, true]) { } else { expect(result.ok).toBe(true); if (!result.ok) throw result.cause; - expect(result.value.subarray(16)).toEqual(Buffer.from("body")); + expect( + Buffer.compare(result.value.subarray(16), Buffer.from("body")), + ).toBe(0); } await reader.close(); expect(attempts).toBe(failClose ? 2 : 1); diff --git a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts index 1e70f61986..898a0995fc 100644 --- a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts @@ -572,7 +572,9 @@ for (const typeCode of ["o", "d"] as const) { path, inventory, ); - expect(await readFile(join(output, realPath))).toEqual(payload); + expect( + Buffer.compare(await readFile(join(output, realPath)), payload), + ).toBe(0); }, ); }); @@ -795,8 +797,11 @@ it.each(["x", "s", "empty-module"])( inventory, ); expect( - (await readFile(join(output, "PYZ.pyz/app.pyc"))).subarray(16), - ).toEqual(Buffer.from("readable module")); + Buffer.compare( + (await readFile(join(output, "PYZ.pyz/app.pyc"))).subarray(16), + Buffer.from("readable module"), + ), + ).toBe(0); }, ); }, @@ -1380,7 +1385,9 @@ it.each([true, false])( source, inventory, ); - expect(await readFile(join(output, path))).toEqual(file.payload); + expect( + Buffer.compare(await readFile(join(output, path)), file.payload), + ).toBe(0); }, ); }, diff --git a/src/artifacts/pyinstaller/PyzArchive.test.ts b/src/artifacts/pyinstaller/PyzArchive.test.ts index 6ea1fedf16..e553589d5b 100644 --- a/src/artifacts/pyinstaller/PyzArchive.test.ts +++ b/src/artifacts/pyinstaller/PyzArchive.test.ts @@ -29,7 +29,7 @@ it("lists modules from a supported PYZ and leaves an unknown magic opaque", asyn ); expect(opaque.ok).toBe(true); if (!opaque.ok || opaque.value.kind !== "opaque") return; - expect(opaque.value.pythonMagic).toEqual(Buffer.alloc(4)); + expect(Buffer.compare(opaque.value.pythonMagic, Buffer.alloc(4))).toBe(0); expect(opaque.value.limitation).toContain("left opaque"); }); diff --git a/src/browser/BrowserScenario.test.ts b/src/browser/BrowserScenario.test.ts index a30c82f85c..16a7b00293 100644 --- a/src/browser/BrowserScenario.test.ts +++ b/src/browser/BrowserScenario.test.ts @@ -102,20 +102,21 @@ describe("browser scenario secret redaction", () => { "raw A/B? remains case-sensitive", ); expect( - secrets.redactBytes( + Buffer.compare( + secrets.redactBytes( + Buffer.concat([ + Buffer.from([0xff, 0]), + Buffer.from("a%2fb%3F"), + Buffer.from([0x80]), + ]), + ), Buffer.concat([ Buffer.from([0xff, 0]), - Buffer.from("a%2fb%3F"), + Buffer.from("[REDACTED:session]"), Buffer.from([0x80]), ]), ), - ).toEqual( - Buffer.concat([ - Buffer.from([0xff, 0]), - Buffer.from("[REDACTED:session]"), - Buffer.from([0x80]), - ]), - ); + ).toBe(0); expect( secrets.sanitizeUrl("https://app.example.test/?token=a%2fb%3F").url, ).toBe("https://app.example.test/?token=[REDACTED:session]"); diff --git a/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts b/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts index ec58d042a9..7989c1a673 100644 --- a/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts +++ b/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts @@ -51,7 +51,9 @@ it("rejects forbidden trailer controls while retaining valid earlier trailers", expect(result.frames[0]?.trailers).toEqual([ { name: "grpc-message", value: "kept" }, ]); - expect(input.subarray(5)).toEqual(Buffer.from(text, "utf8")); + expect(Buffer.compare(input.subarray(5), Buffer.from(text, "utf8"))).toBe( + 0, + ); } }); @@ -116,7 +118,7 @@ it.each(["grpc-web", "grpc-web-text"] as const)( ], }); expect(result.examined_bytes).toBe(binary.length); - expect(binary.subarray(-trailer.length)).toEqual(trailer); + expect(Buffer.compare(binary.subarray(-trailer.length), trailer)).toBe(0); }, ); diff --git a/src/browser/network/PlaywrightScenarioNetwork.test.ts b/src/browser/network/PlaywrightScenarioNetwork.test.ts index ec7c18186c..d74a9a14e5 100644 --- a/src/browser/network/PlaywrightScenarioNetwork.test.ts +++ b/src/browser/network/PlaywrightScenarioNetwork.test.ts @@ -401,13 +401,16 @@ it("preserves duplicate headers and binary bytes, redacting credentials and decl ); if (content?.kind !== "network-content" || content.body.state !== "captured") throw new Error("Missing response"); - expect(Buffer.from(content.body.content, "base64")).toEqual( - Buffer.concat([ - Buffer.from([0xff, 0x00]), - Buffer.from("[REDACTED:declared]"), - Buffer.from([0xfe]), - ]), - ); + expect( + Buffer.compare( + Buffer.from(content.body.content, "base64"), + Buffer.concat([ + Buffer.from([0xff, 0x00]), + Buffer.from("[REDACTED:declared]"), + Buffer.from([0xfe]), + ]), + ), + ).toBe(0); expect(content.body.redacted).toBe(true); expect(content.headers).toEqual({ state: "captured", diff --git a/src/domain/apple/plistValue.test.ts b/src/domain/apple/plistValue.test.ts index a06899531e..01e9ce026b 100644 --- a/src/domain/apple/plistValue.test.ts +++ b/src/domain/apple/plistValue.test.ts @@ -236,5 +236,5 @@ it.each([ expect(projected.nonFiniteRealCount).toBe(4); expect(projected.negativeZeroRealCount).toBe(1); expect(observed).toHaveLength(5); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); }); diff --git a/src/domain/native/nativeAotPe.discoveryAccounting.test.ts b/src/domain/native/nativeAotPe.discoveryAccounting.test.ts index 6985e152af..9a40ff0ce3 100644 --- a/src/domain/native/nativeAotPe.discoveryAccounting.test.ts +++ b/src/domain/native/nativeAotPe.discoveryAccounting.test.ts @@ -57,7 +57,7 @@ describe("NativeAOT candidate work accounting", () => { }, }); expect(summary.coverage.work_units_examined).toBeLessThan(64 * 1024); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); }); it.each(["slots", "interfaces"] as const)( @@ -85,7 +85,7 @@ describe("NativeAOT candidate work accounting", () => { scan_bytes_omitted: 0, }, }); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); }, ); @@ -112,6 +112,6 @@ describe("NativeAOT candidate work accounting", () => { "0x1400022c0", "0x140002400", ]); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); }); }); diff --git a/src/domain/native/nativeAotPe.sectionRanges.test.ts b/src/domain/native/nativeAotPe.sectionRanges.test.ts index 6efaf1caf6..75d482ccd0 100644 --- a/src/domain/native/nativeAotPe.sectionRanges.test.ts +++ b/src/domain/native/nativeAotPe.sectionRanges.test.ts @@ -27,7 +27,7 @@ it.each(["before-data", "after-data"] as const)( expect( result.readTypeDetail("0x140002280", NATIVE_AOT_MAX_REPORT_BYTES), ).toBeUndefined(); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); }, ); @@ -110,7 +110,7 @@ it("preserves source section order while admitting a unique unsorted mapping", ( status: "complete", method_tables: 3, }); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); }); it("does not confuse raw alignment padding with virtual section ownership", () => { diff --git a/src/domain/native/nativeAotPe.test.ts b/src/domain/native/nativeAotPe.test.ts index 282dc1498e..ebcdccf20c 100644 --- a/src/domain/native/nativeAotPe.test.ts +++ b/src/domain/native/nativeAotPe.test.ts @@ -64,7 +64,7 @@ describe("NativeAOT indexed MethodTable closure", () => { "0x1400022c0", ...expected, ]); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); }, ); @@ -91,7 +91,7 @@ describe("read-only NativeAOT PE parser", () => { SUMMARY_BUDGET, "0x140000000", ); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); expect(parsed.summary).toMatchObject({ format: "dotnet-nativeaot", status: "complete", @@ -227,7 +227,7 @@ describe("read-only NativeAOT PE parser", () => { nativeAotPeDigest(bytes), SUMMARY_BUDGET, ); - expect(bytes).toEqual(before); + expect(Buffer.compare(bytes, before)).toBe(0); expect(parsed.summary).toMatchObject({ status: "partial", derived_overlay: null, diff --git a/src/dotnet/ManagedMetadataHeaps.test.ts b/src/dotnet/ManagedMetadataHeaps.test.ts index 9bb3321210..469f547b34 100644 --- a/src/dotnet/ManagedMetadataHeaps.test.ts +++ b/src/dotnet/ManagedMetadataHeaps.test.ts @@ -47,9 +47,12 @@ describe("managed metadata heaps", () => { blob: null, } as unknown as ManagedMetadataLayout; - expect(readMetadataBlob(Buffer.alloc(0), layout, 0, 0)).toEqual( - Buffer.alloc(0), - ); + expect( + Buffer.compare( + readMetadataBlob(Buffer.alloc(0), layout, 0, 0), + Buffer.alloc(0), + ), + ).toBe(0); expect(() => readMetadataBlob(Buffer.alloc(0), layout, 1, 0)).toThrowError( expect.objectContaining({ issue: expect.objectContaining({ diff --git a/src/dotnet/ManagedPeFixtureLayout.test.ts b/src/dotnet/ManagedPeFixtureLayout.test.ts index 99d30a77c6..ecf2fa32fc 100644 --- a/src/dotnet/ManagedPeFixtureLayout.test.ts +++ b/src/dotnet/ManagedPeFixtureLayout.test.ts @@ -41,11 +41,20 @@ describe("managed PE fixture layout", () => { }); expect( - image.subarray(rawOffset(0x2800), rawOffset(0x2800) + firstBody.length), - ).toEqual(firstBody); + Buffer.compare( + image.subarray(rawOffset(0x2800), rawOffset(0x2800) + firstBody.length), + firstBody, + ), + ).toBe(0); expect( - image.subarray(rawOffset(0x2840), rawOffset(0x2840) + secondBody.length), - ).toEqual(secondBody); + Buffer.compare( + image.subarray( + rawOffset(0x2840), + rawOffset(0x2840) + secondBody.length, + ), + secondBody, + ), + ).toBe(0); const inspected = inspectManagedMembersBytes( image, managedPeFixtureTarget(image), @@ -92,11 +101,14 @@ describe("managed PE fixture layout", () => { expect(layout.cli.resources.size).toBe(resourceData.length + 4); expect(image.readUInt32LE(directoryOffset)).toBe(resourceData.length); expect( - image.subarray( - directoryOffset + 4, - directoryOffset + 4 + resourceData.length, + Buffer.compare( + image.subarray( + directoryOffset + 4, + directoryOffset + 4 + resourceData.length, + ), + resourceData, ), - ).toEqual(resourceData); + ).toBe(0); expect(inspection.classification.status).toBe("managed"); expect(inspection.resources).toMatchObject([ { diff --git a/src/electron/ElectronFuseArchitecture.test.ts b/src/electron/ElectronFuseArchitecture.test.ts index aeafce7c36..b9df79e185 100644 --- a/src/electron/ElectronFuseArchitecture.test.ts +++ b/src/electron/ElectronFuseArchitecture.test.ts @@ -114,5 +114,5 @@ const assertArchitecture = async ( expect( report.slices[0]?.sentinels[0]?.fuses.map((fuse) => fuse.raw_byte), ).toEqual(OBSIDIAN_FUSE_BYTES); - expect(await readFile(path)).toEqual(image.bytes); + expect(Buffer.compare(await readFile(path), image.bytes)).toBe(0); }; diff --git a/src/flutter/DartElf.test.ts b/src/flutter/DartElf.test.ts index 573121171a..c726225078 100644 --- a/src/flutter/DartElf.test.ts +++ b/src/flutter/DartElf.test.ts @@ -112,7 +112,7 @@ describe("readElfSymbol", () => { const image = strippedImageWithSections([[tag, 0xffffffffffffffffn]]); const original = Buffer.from(image); expect(await readElfSymbols(image, names)).toEqual(control); - expect(image).toEqual(original); + expect(Buffer.compare(image, original)).toBe(0); }, ); @@ -202,7 +202,7 @@ describe("dynamic symbols without a SHT_DYNSYM section", () => { expect(control.symbols).toHaveLength(2); const before = Buffer.from(image); expect(await readElfSymbols(image, names)).toEqual(control); - expect(image).toEqual(before); + expect(Buffer.compare(image, before)).toBe(0); }, ); diff --git a/src/ghidra/GhidraLaunchFailure.test.ts b/src/ghidra/GhidraLaunchFailure.test.ts index 425f817f03..7f6d226de1 100644 --- a/src/ghidra/GhidraLaunchFailure.test.ts +++ b/src/ghidra/GhidraLaunchFailure.test.ts @@ -136,10 +136,11 @@ describe("Ghidra launch failure diagnostics", () => { expect(outcome?.["stderrTruncated"]).toBe(true); expect(outcome?.["outputTruncated"]).toBe(false); expect(outcome?.["stderrBytes"]).toBe(oversized.length); - expect((outcome?.["stderr"] as string).length).toBeLessThanOrEqual( - EXPECTED_OUTPUT_BOUND_BYTES, - ); - expect(outcome?.["stderr"] as string).not.toContain("UNRETAINED-TAIL"); + const stderr = outcome?.["stderr"]; + if (typeof stderr !== "string") + throw new Error("Expected launcher stderr"); + expect(stderr.length).toBeLessThanOrEqual(EXPECTED_OUTPUT_BOUND_BYTES); + expect(stderr).not.toContain("UNRETAINED-TAIL"); } finally { await rm(javaHome, { recursive: true, force: true }); await rm(runtimeRoot, { recursive: true, force: true }); diff --git a/src/ghidra/extensions/GhidraExtensions.test.ts b/src/ghidra/extensions/GhidraExtensions.test.ts index 61edea4beb..2df197d977 100644 --- a/src/ghidra/extensions/GhidraExtensions.test.ts +++ b/src/ghidra/extensions/GhidraExtensions.test.ts @@ -161,9 +161,12 @@ describe("optional Ghidra extension boundary", () => { resources, ); expect(snapshots[0]?.path).toBe(join(runtime, "extensions/nativeaot.jar")); - expect(await readFile(join(runtime, "extensions/nativeaot.jar"))).toEqual( - bytes, - ); + expect( + Buffer.compare( + await readFile(join(runtime, "extensions/nativeaot.jar")), + bytes, + ), + ).toBe(0); await expect( snapshotGhidraExtensions([extension], runtime, resources), ).rejects.toMatchObject({ diff --git a/src/gpu/GpuPayloadCodec.test.ts b/src/gpu/GpuPayloadCodec.test.ts index 465fc725c4..6b1f11f9aa 100644 --- a/src/gpu/GpuPayloadCodec.test.ts +++ b/src/gpu/GpuPayloadCodec.test.ts @@ -7,9 +7,12 @@ it.each(["zlib", "zstd"] as const)( const source = Buffer.from("kernel"), encode = codec === "zstd" ? zstdCompressSync : deflateSync, compressed = encode(source); - expect(await decodeGpuPayload(compressed, codec, source.length)).toEqual( - source, - ); + expect( + Buffer.compare( + await decodeGpuPayload(compressed, codec, source.length), + source, + ), + ).toBe(0); await expect( decodeGpuPayload( Buffer.concat([compressed, Buffer.from("junk")]), diff --git a/src/hopper/HopperImageCopy.test.ts b/src/hopper/HopperImageCopy.test.ts index a0e57bfc35..1a3cbc01ea 100644 --- a/src/hopper/HopperImageCopy.test.ts +++ b/src/hopper/HopperImageCopy.test.ts @@ -37,8 +37,8 @@ it("whole executable copying rejects pre-copy replacement and retains captured b ); expect(captured).toEqual({ ok: true, value: capturedPath }); await writeFile(source, replacement); - expect(await readFile(capturedPath)).toEqual(selected); - expect(await readFile(source)).toEqual(replacement); + expect(Buffer.compare(await readFile(capturedPath), selected)).toBe(0); + expect(Buffer.compare(await readFile(source), replacement)).toBe(0); } finally { await resources.close(); } diff --git a/src/hopper/HopperMachOImage.test.ts b/src/hopper/HopperMachOImage.test.ts index 3a90caa6c9..fcac956979 100644 --- a/src/hopper/HopperMachOImage.test.ts +++ b/src/hopper/HopperMachOImage.test.ts @@ -385,7 +385,7 @@ it("copies exactly the selected FAT32 slice bytes and verifies the source", asyn arm64eOffset, arm64eOffset + FAT32_SLICE_SIZE, ); - expect(await readFile(destination)).toEqual(copied); + expect(Buffer.compare(await readFile(destination), copied)).toBe(0); }); it("retains the caller reason when a FAT copy is cancelled after its final write", async () => { @@ -421,12 +421,15 @@ it("retains the caller reason when a FAT copy is cancelled after its final write expect(prepared.error._tag).toBe("HopperCancelledError"); expect(prepared.error.cause).toBe(reason); // The private session directory remains the owner of partial output. - await expect(readFile(destination)).resolves.toEqual( - fixture.container.subarray( - fixture.sliceOffsets[0], - fixture.sliceOffsets[0]! + FAT32_SLICE_SIZE, + expect( + Buffer.compare( + await readFile(destination), + fixture.container.subarray( + fixture.sliceOffsets[0], + fixture.sliceOffsets[0]! + FAT32_SLICE_SIZE, + ), ), - ); + ).toBe(0); }); it("retains a failed prepared-image close through launcher cleanup and blocks another launch", async () => { @@ -522,9 +525,12 @@ it("retains a failed prepared-image close through launcher cleanup and blocks an expect(initial.error.cleanupIncomplete).toBe(false); expect(blocked.error.cleanupIncomplete).toBe(false); expect(cleanupPending.error.cleanupIncomplete).toBe(false); - await expect( - readFile(join(sessionDirectory, "image.macho")), - ).resolves.toEqual(bytes); + expect( + Buffer.compare( + await readFile(join(sessionDirectory, "image.macho")), + bytes, + ), + ).toBe(0); }); it("observes caller cancellation that arrives during final source-handle cleanup", async () => { @@ -574,9 +580,9 @@ it("observes caller cancellation that arrives during final source-handle cleanup if (prepared.ok) throw new Error("Finalization cancellation was ignored"); expect(prepared.error._tag).toBe("HopperCancelledError"); expect(prepared.error.cause).toBe(reason); - await expect(readFile(join(directory, "image.macho"))).resolves.toEqual( - bytes, - ); + expect( + Buffer.compare(await readFile(join(directory, "image.macho")), bytes), + ).toBe(0); }); it("rejects FAT32 preparation when the source table changed after resolution", async () => { @@ -646,7 +652,7 @@ it("copies an admitted thin Mach-O through the owned verified-image path", async ); if (!prepared.ok) throw new Error(prepared.error.userMessage); expect(prepared.value).toBe(destination); - expect(await readFile(destination)).toEqual(bytes); + expect(Buffer.compare(await readFile(destination), bytes)).toBe(0); expect(target.path).toBe(path); }); @@ -777,6 +783,6 @@ it("retains an owned partial thin image and exact cancellation reason after a wr if (prepared.ok) throw new Error("Cancelled partial copy was accepted"); expect(prepared.error._tag).toBe("HopperCancelledError"); expect(prepared.error.cause).toBe(reason); - await expect(readFile(destination)).resolves.toEqual(partial); + expect(Buffer.compare(await readFile(destination), partial)).toBe(0); await expect(access(destination)).resolves.toBeUndefined(); }); diff --git a/src/ida/IdaSessionClient.test.ts b/src/ida/IdaSessionClient.test.ts index a7d24f9ac6..59908c79ff 100644 --- a/src/ida/IdaSessionClient.test.ts +++ b/src/ida/IdaSessionClient.test.ts @@ -1147,7 +1147,9 @@ describe("headless IDA session ownership and cleanup", () => { }); if (!result.ok) throw result.error; expect(producer.inputPath).not.toBe(target.path); - expect(await readFile(producer.inputPath)).toEqual(original); + expect(Buffer.compare(await readFile(producer.inputPath), original)).toBe( + 0, + ); expect( functionDossierSchema .parse(result.value.result) @@ -1167,7 +1169,7 @@ describe("headless IDA session ownership and cleanup", () => { producer.calls.find(({ name }) => name === "idb_close")?.args.save, ).toBe(false); expect(await readdir(root)).toEqual(["target.elf"]); - expect(await readFile(target.path)).toEqual(original); + expect(Buffer.compare(await readFile(target.path), original)).toBe(0); }); it("retains the private workspace and reports cleanup failure when worker release is unverified", async () => { const { producer, client, root } = await fixture("headless"); diff --git a/src/native/parsers/plist.test.ts b/src/native/parsers/plist.test.ts index 1e6ef21077..69f24d1d7d 100644 --- a/src/native/parsers/plist.test.ts +++ b/src/native/parsers/plist.test.ts @@ -106,7 +106,7 @@ const parseWithoutMutatingSource = (bytes: Buffer) => { const parsed = parsePlistBinary(bytes); expect(parsed.ok).toBe(true); if (!parsed.ok) throw parsed.error; - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); return parsed.value; }; diff --git a/src/native/pwndbg/CoreFileMappingAuthority.test.ts b/src/native/pwndbg/CoreFileMappingAuthority.test.ts index 25dc0bae61..4f7e12c198 100644 --- a/src/native/pwndbg/CoreFileMappingAuthority.test.ts +++ b/src/native/pwndbg/CoreFileMappingAuthority.test.ts @@ -43,7 +43,7 @@ describe("core-only NT_FILE authority", () => { expect(result.ok).toBe( ["/usr/bin/app", "//usr/lib/libc.so", "/a/.../b"].includes(path), ); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); if (!result.ok) expect(result.error).toMatchObject({ _tag: "AnalysisCapabilityUnavailableError", diff --git a/src/process/ExecFileOutput.test.ts b/src/process/ExecFileOutput.test.ts index 21f2f826ba..4aacc960c4 100644 --- a/src/process/ExecFileOutput.test.ts +++ b/src/process/ExecFileOutput.test.ts @@ -51,8 +51,10 @@ describe("execFileOutput", () => { "process.stdout.write(Buffer.from([0x00, 0xff, 0x41])); process.stderr.write(Buffer.from([0xfe]));", ]); - expect(result.stdout).toEqual(Buffer.from([0x00, 0xff, 0x41])); - expect(result.stderr).toEqual(Buffer.from([0xfe])); + expect(Buffer.compare(result.stdout, Buffer.from([0x00, 0xff, 0x41]))).toBe( + 0, + ); + expect(Buffer.compare(result.stderr, Buffer.from([0xfe]))).toBe(0); }); it("reports raw byte counts when UTF-8 decoding changes the captured text", async () => { diff --git a/src/reference/ReferenceSourceReaderFile.test.ts b/src/reference/ReferenceSourceReaderFile.test.ts index 9f4a20dcd1..205e2f8904 100644 --- a/src/reference/ReferenceSourceReaderFile.test.ts +++ b/src/reference/ReferenceSourceReaderFile.test.ts @@ -26,7 +26,9 @@ it("retains all bytes across short reads through the declared size", async () => expectedSize: BigInt(bytes.length), }); if (result.status === "failed") throw result.entry; - expect(Buffer.concat(result.chunks, result.total)).toEqual(bytes); + expect( + Buffer.compare(Buffer.concat(result.chunks, result.total), bytes), + ).toBe(0); }); it("reports growth instead of continuing to consume a changed file", async () => { diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index 5cbb8cf284..bd70e0894c 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -327,7 +327,9 @@ it.each([ expect(report.tool_profile.commands[0]?.arguments).toContain( "--ignore-custom-section-errors", ); - expect(await readFile(fixture.path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(fixture.path), fixture.bytes)).toBe( + 0, + ); expect(report.headers).toBe( dump.slice(0, dump.indexOf("Section Details:\n")), ); @@ -722,7 +724,12 @@ const launchWabtOutcome = async ( const { bytes, dump, controller } = fixture; const version = input.arguments[0] === "--version"; if (!version) { - expect(await readFile(join(input.cwd ?? "", "module.wasm"))).toEqual(bytes); + expect( + Buffer.compare( + await readFile(join(input.cwd ?? "", "module.wasm")), + bytes, + ), + ).toBe(0); if (mode === "changed-snapshot") { const snapshot = join(input.cwd ?? "", "module.wasm"); await chmod(snapshot, 0o600); @@ -821,7 +828,7 @@ it("enforces a selected command deadline and completes a corrected retry after c const report = wasmArtifactSchema.parse(corrected.value.result); expect(report.tool_profile.timeout_ms).toBe(2_000); expect(report.limitations.join(" ")).not.toContain("30-second"); - await expect(readFile(fixture.path)).resolves.toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(fixture.path), fixture.bytes)).toBe(0); }); const glueSourcesForOutcome = (mode: OutcomeMode) => { @@ -1006,7 +1013,7 @@ const runOwnedArtifactOutcome = async ( ); await assertSnapshotCleanupOutcome(mode, result, provider, runtimePath); await expect(access(runtimePath)).rejects.toMatchObject({ code: "ENOENT" }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); if (acquiredPid !== undefined) { const pid = acquiredPid; expect(() => process.kill(pid, 0)).toThrow(); diff --git a/src/wasm/WabtString.test.ts b/src/wasm/WabtString.test.ts index bdbaf40afe..a0c2d7cea2 100644 --- a/src/wasm/WabtString.test.ts +++ b/src/wasm/WabtString.test.ts @@ -17,7 +17,9 @@ it("frames escaped quotes and trailing backslashes without consuming the next fo const encoded = JSON.stringify(value); const source = encoded + ' (export "next" (func 0))'; expect(watStringEnd(source, 1)).toBe(encoded.length); - expect(watString(source, 1).bytes).toEqual(Buffer.from(value)); + expect( + Buffer.compare(watString(source, 1).bytes, Buffer.from(value)), + ).toBe(0); } } }); diff --git a/tests/acceptance/apkZip64FramingCli.test.ts b/tests/acceptance/apkZip64FramingCli.test.ts index ea02f724a7..e75291f992 100644 --- a/tests/acceptance/apkZip64FramingCli.test.ts +++ b/tests/acceptance/apkZip64FramingCli.test.ts @@ -58,7 +58,7 @@ const expectUnsignedZip64 = async ( runtime_execution: "not-performed", }); } - expect(await readFile(path)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(path), Buffer.from(bytes))).toBe(0); }; const expectRefusal = async ( @@ -75,7 +75,7 @@ const expectRefusal = async ( const mcpError = parseMcpToolError(response).error; expect(mcpError).toMatchObject(expected); expect(mcpError.message).toBe(cliError.message); - expect(await readFile(path)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(path), Buffer.from(bytes))).toBe(0); }; cliTest( diff --git a/tests/acceptance/applications/appleBundleTraversalCoverage.test.ts b/tests/acceptance/applications/appleBundleTraversalCoverage.test.ts index d368d5c479..e2a0ce2cb8 100644 --- a/tests/acceptance/applications/appleBundleTraversalCoverage.test.ts +++ b/tests/acceptance/applications/appleBundleTraversalCoverage.test.ts @@ -84,7 +84,7 @@ cliTest.skipIf(process.platform === "win32")( "Contents/Resources/Panel.nib: Symlink target was not followed or disclosed.", ); } - expect(await readFile(link)).toEqual(fixture.nib); + expect(Buffer.compare(await readFile(link), fixture.nib)).toBe(0); }, ); diff --git a/tests/acceptance/applications/binaryPlistDates.test.ts b/tests/acceptance/applications/binaryPlistDates.test.ts index bdc78b8f4d..80f756eeca 100644 --- a/tests/acceptance/applications/binaryPlistDates.test.ts +++ b/tests/acceptance/applications/binaryPlistDates.test.ts @@ -204,12 +204,19 @@ cliTest( ), ).toEqual(ibResult); - expect(await readFile(archivePath)).toEqual(bytes); - expect(await readFile(malformedArchivePath)).toEqual(malformedBytes); + expect(Buffer.compare(await readFile(archivePath), bytes)).toBe(0); expect( - await readFile(join(malformedApp, "Contents/Resources/Main.nib")), - ).toEqual(bytes); - expect(await readFile(malformedNib)).toEqual(malformedBytes); + Buffer.compare(await readFile(malformedArchivePath), malformedBytes), + ).toBe(0); + expect( + Buffer.compare( + await readFile(join(malformedApp, "Contents/Resources/Main.nib")), + bytes, + ), + ).toBe(0); + expect(Buffer.compare(await readFile(malformedNib), malformedBytes)).toBe( + 0, + ); expect(createHash("sha256").update(bytes).digest("hex")).toBe( ibResult.documents[0]?.archive_sha256, ); @@ -265,7 +272,7 @@ cliTest( parseEvidence(validated).normalized_result, ); expect(mcpResult).toEqual(cliResult); - expect(await readFile(archivePath)).toEqual(bytes); + expect(Buffer.compare(await readFile(archivePath), bytes)).toBe(0); }, 60_000, ); diff --git a/tests/acceptance/applications/hopperPublisherRejectionCli.test.ts b/tests/acceptance/applications/hopperPublisherRejectionCli.test.ts index 9e22759839..1da519cac5 100644 --- a/tests/acceptance/applications/hopperPublisherRejectionCli.test.ts +++ b/tests/acceptance/applications/hopperPublisherRejectionCli.test.ts @@ -61,6 +61,6 @@ cliTest }, }); await expect(access(marker)).rejects.toMatchObject({ code: "ENOENT" }); - expect(await readFile(target)).toEqual(original); + expect(Buffer.compare(await readFile(target), original)).toBe(0); }, ); diff --git a/tests/acceptance/applications/interfaceBuilderUIKitKeyedArchive.test.ts b/tests/acceptance/applications/interfaceBuilderUIKitKeyedArchive.test.ts index 5f86aa9ea9..0b137347b8 100644 --- a/tests/acceptance/applications/interfaceBuilderUIKitKeyedArchive.test.ts +++ b/tests/acceptance/applications/interfaceBuilderUIKitKeyedArchive.test.ts @@ -243,7 +243,7 @@ cliTest.for(["xml", "binary"] as const)( }), ); expect(report.graph.truncated).toBe(false); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); diff --git a/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts b/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts index 9fc6f49f6f..1a8eb73ba1 100644 --- a/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts +++ b/tests/acceptance/applications/javascriptSourceSelectionCli.test.ts @@ -92,7 +92,9 @@ cliTest.for([ expect(properties?.parse_status).toBe( path === "main.mjs" ? "complete" : null, ); - expect(await readFile(join(directory, path))).toEqual(bytes); + expect(Buffer.compare(await readFile(join(directory, path)), bytes)).toBe( + 0, + ); } const fullResponse = await cli.run({ arguments: [command, input, "--format", "json"], diff --git a/tests/acceptance/applications/jvmAdmissionIdentity.test.ts b/tests/acceptance/applications/jvmAdmissionIdentity.test.ts index e13ec98933..4aaf6148b3 100644 --- a/tests/acceptance/applications/jvmAdmissionIdentity.test.ts +++ b/tests/acceptance/applications/jvmAdmissionIdentity.test.ts @@ -28,12 +28,12 @@ cliTest( }); expect(analyzed.exitCode, analyzed.stderr).toBe(1); expect(analyzed.json).toMatchObject(refusal); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); const opened = await call("open_binary", { path }); expect(opened.isError, JSON.stringify(opened)).toBe(true); expect(parseMcpToolError(opened).error).toMatchObject(refusal); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } }, ); diff --git a/tests/acceptance/applications/keyedArchiveScalarKinds.test.ts b/tests/acceptance/applications/keyedArchiveScalarKinds.test.ts index 7d9322d523..8e25eb45b9 100644 --- a/tests/acceptance/applications/keyedArchiveScalarKinds.test.ts +++ b/tests/acceptance/applications/keyedArchiveScalarKinds.test.ts @@ -154,7 +154,7 @@ cliTest.for(["xml", "binary"] as const)( status: "resolved", }), ]); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); @@ -243,7 +243,7 @@ cliTest.for(["xml", "binary"] as const)( expect(result.limitations).toContain( '1 negative-zero real value(s) are reported as { "$plist_type": "real", "value": "-0" } because a JSON number cannot carry the sign of zero.', ); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); @@ -346,7 +346,7 @@ cliTest("preserves binary IEEE non-finite scalar identity", async ({ cli }) => { parseEvidence(mcp.structuredContent).normalized_result, ), ).toEqual(result); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); cliTest( @@ -419,6 +419,6 @@ cliTest( expect(result.limitations).not.toContainEqual( expect.stringContaining('"value": null'), ); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); diff --git a/tests/acceptance/applications/keyedArchiveXmlNumbers.test.ts b/tests/acceptance/applications/keyedArchiveXmlNumbers.test.ts index 418f86845c..0c10ff8019 100644 --- a/tests/acceptance/applications/keyedArchiveXmlNumbers.test.ts +++ b/tests/acceptance/applications/keyedArchiveXmlNumbers.test.ts @@ -130,7 +130,7 @@ cliTest.for([ /non-finite|completely observed/u.test(note), ), ).toBe(false); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); @@ -178,7 +178,7 @@ cliTest( }, }, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } }, ); diff --git a/tests/acceptance/applications/keyedArchiveXmlScalars.test.ts b/tests/acceptance/applications/keyedArchiveXmlScalars.test.ts index 301bfb1589..73da482ced 100644 --- a/tests/acceptance/applications/keyedArchiveXmlScalars.test.ts +++ b/tests/acceptance/applications/keyedArchiveXmlScalars.test.ts @@ -105,6 +105,6 @@ cliTest( parseEvidence(reply.structuredContent).normalized_result, ), ).toEqual(result); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); diff --git a/tests/acceptance/applications/mobileDirectoryComponents.test.ts b/tests/acceptance/applications/mobileDirectoryComponents.test.ts index ee45298353..fab73ee2b5 100644 --- a/tests/acceptance/applications/mobileDirectoryComponents.test.ts +++ b/tests/acceptance/applications/mobileDirectoryComponents.test.ts @@ -207,7 +207,7 @@ cliTest.for(fixtures)( mode === "files" ? childPackages : [], ); } - expect(await readFile(path)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(path), Buffer.from(bytes))).toBe(0); } }, ); @@ -351,7 +351,7 @@ cliTest.for([ } } if (encrypted) expect(Object.values(graph.components).flat()).toEqual([]); - expect(await readFile(path)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(path), Buffer.from(bytes))).toBe(0); } }, ); diff --git a/tests/acceptance/applications/nativePlistBinaryScalars.test.ts b/tests/acceptance/applications/nativePlistBinaryScalars.test.ts index c68cd82cce..c14bea7051 100644 --- a/tests/acceptance/applications/nativePlistBinaryScalars.test.ts +++ b/tests/acceptance/applications/nativePlistBinaryScalars.test.ts @@ -174,8 +174,8 @@ describe.skipIf(process.platform !== "darwin")( provenance: [{ tool: "file" }], limitations: [], }); - expect(await readFile(binaryPath)).toEqual(binaryBytes); - expect(await readFile(xmlPath)).toEqual(xmlBytes); + expect(Buffer.compare(await readFile(binaryPath), binaryBytes)).toBe(0); + expect(Buffer.compare(await readFile(xmlPath), xmlBytes)).toBe(0); }, 60_000, ); diff --git a/tests/acceptance/applications/nativePlistNesting.test.ts b/tests/acceptance/applications/nativePlistNesting.test.ts index a7f4488fcc..faa2624865 100644 --- a/tests/acceptance/applications/nativePlistNesting.test.ts +++ b/tests/acceptance/applications/nativePlistNesting.test.ts @@ -89,7 +89,7 @@ describe.skipIf(process.platform !== "darwin")( environment, }); const response = await call("inspect_plist", { path }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); if (depth === 999) { expect(output.exitCode, output.stderr).toBe(1); const expectedFailure = { diff --git a/tests/acceptance/applications/nibFieldIdentity.test.ts b/tests/acceptance/applications/nibFieldIdentity.test.ts index 166583ffbb..44c31f5315 100644 --- a/tests/acceptance/applications/nibFieldIdentity.test.ts +++ b/tests/acceptance/applications/nibFieldIdentity.test.ts @@ -194,7 +194,7 @@ cliTest( expect(report.limitations).toContainEqual( expect.stringContaining("NSString NS.bytes is not valid UTF-8"), ); - expect(await readFile(path)).toEqual(archive); + expect(Buffer.compare(await readFile(path), archive)).toBe(0); }, 60_000, ); diff --git a/tests/acceptance/applications/packageUrlUnicodeCli.test.ts b/tests/acceptance/applications/packageUrlUnicodeCli.test.ts index 5dcdd46db6..40dc5ec6b3 100644 --- a/tests/acceptance/applications/packageUrlUnicodeCli.test.ts +++ b/tests/acceptance/applications/packageUrlUnicodeCli.test.ts @@ -277,6 +277,6 @@ cliTest.for(["CLI", "stdio MCP"] as const)( }), ); } - expect(await readFile(archive)).toEqual(bytes); + expect(Buffer.compare(await readFile(archive), bytes)).toBe(0); }, ); diff --git a/tests/acceptance/applications/peResources.test.ts b/tests/acceptance/applications/peResources.test.ts index 8e50fe2a5b..154101e0d4 100644 --- a/tests/acceptance/applications/peResources.test.ts +++ b/tests/acceptance/applications/peResources.test.ts @@ -132,7 +132,7 @@ const expectPeWordAlignedRecordsAcrossAdapters = async (input: { contract.outputSchema.parse(response.structuredContent) .normalized_result, ).toEqual(evidence.normalized_result); - expect(await readFile(path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(path), fixture.bytes)).toBe(0); } } }; @@ -227,7 +227,7 @@ const expectPeMappingOwnershipAcrossAdapters = async ( }, ], }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } }; @@ -308,7 +308,7 @@ const expectPeInventoriesAcrossAdapters = async ( expect(contract.outputSchema.parse(JSON.parse(text.text)).evidence_id).toBe( evidence.evidence_id, ); - expect(await readFile(path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(path), fixture.bytes)).toBe(0); } }; @@ -406,7 +406,7 @@ const expectPeEmptyBranchesAcrossAdapters = async ( expect(evidence.subject?.digest?.sha256).toBe( createHash("sha256").update(fixture.bytes).digest("hex"), ); - expect(await readFile(path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(path), fixture.bytes)).toBe(0); } }; diff --git a/tests/acceptance/applications/resourceOnlyPlistSelection.test.ts b/tests/acceptance/applications/resourceOnlyPlistSelection.test.ts index e5a22ed9be..69103a471a 100644 --- a/tests/acceptance/applications/resourceOnlyPlistSelection.test.ts +++ b/tests/acceptance/applications/resourceOnlyPlistSelection.test.ts @@ -69,7 +69,11 @@ it.skipIf(process.platform !== "darwin")( code: "invalid_request", details: { issues: [{ path: ["path"] }] }, }); - expect(await readFile(sourcePath)).toEqual(Buffer.from(source)); - expect(await readFile(decoyPath)).toEqual(Buffer.from(decoy)); + expect( + Buffer.compare(await readFile(sourcePath), Buffer.from(source)), + ).toBe(0); + expect(Buffer.compare(await readFile(decoyPath), Buffer.from(decoy))).toBe( + 0, + ); }, ); diff --git a/tests/acceptance/artifactExtractionCoverage.test.ts b/tests/acceptance/artifactExtractionCoverage.test.ts index 9a0ff8ce4b..9047b52f04 100644 --- a/tests/acceptance/artifactExtractionCoverage.test.ts +++ b/tests/acceptance/artifactExtractionCoverage.test.ts @@ -81,7 +81,9 @@ cliTest.skipIf(process.platform !== "linux").for([false, true])( result.artifacts.map(({ relative_path }) => relative_path), ).toEqual([...files.keys()]); for (const [path, bytes] of files) - expect(await readFile(join(result.output_root, path))).toEqual(bytes); + expect( + Buffer.compare(await readFile(join(result.output_root, path)), bytes), + ).toBe(0); for (const limitations of [result.limitations, evidence.limitations]) { expect( limitations.some((value) => value.includes("not valid UTF-8")), @@ -171,7 +173,9 @@ cliTest.for([false, true])( result.artifacts.map(({ relative_path }) => relative_path), ).toEqual(files.map(([path]) => path)); for (const [path, bytes] of files) { - expect(await readFile(join(result.output_root, path))).toEqual(bytes); + expect( + Buffer.compare(await readFile(join(result.output_root, path)), bytes), + ).toBe(0); expect( result.artifacts.find((file) => file.relative_path === path), ).toMatchObject({ diff --git a/tests/acceptance/devicetreeCli.test.ts b/tests/acceptance/devicetreeCli.test.ts index 07396f9061..b82039b3ce 100644 --- a/tests/acceptance/devicetreeCli.test.ts +++ b/tests/acceptance/devicetreeCli.test.ts @@ -38,7 +38,7 @@ cliTest("inspects a relative DTB and preserves the source", async ({ cli }) => { summary: { model: "Test board" }, coverage: { examined_nodes: 5 }, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); cliTest( "does not resolve through a legacy phandle when the canonical value is malformed", diff --git a/tests/acceptance/elfNotesCli.test.ts b/tests/acceptance/elfNotesCli.test.ts index 02bca6dddf..13b31943a3 100644 --- a/tests/acceptance/elfNotesCli.test.ts +++ b/tests/acceptance/elfNotesCli.test.ts @@ -51,7 +51,7 @@ cliTest( expect(evidence.normalized_result.entries[0]?.gnu?.build_id_hex).toBe( "0123456789abcdef0123456789abcdef01234567", ); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } }, ); @@ -82,7 +82,7 @@ cliTest( examined_tables: 1, examined_entries: 1, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); @@ -138,7 +138,7 @@ cliTest( "Compressed ELF note sections are unsupported", ), }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); diff --git a/tests/acceptance/erlangBeamCli.test.ts b/tests/acceptance/erlangBeamCli.test.ts index e0fbf2ba16..36cfdd4676 100644 --- a/tests/acceptance/erlangBeamCli.test.ts +++ b/tests/acceptance/erlangBeamCli.test.ts @@ -39,9 +39,12 @@ const extendedFirstAtomLength = (): Buffer => { const payloadOffset = chunkOffset + 8; const payloadSize = control.readUInt32BE(chunkOffset + 4); const payloadEnd = payloadOffset + payloadSize; - expect(control.subarray(payloadOffset + 4, payloadOffset + 6)).toEqual( - Buffer.from([0x08, 0x10]), - ); + expect( + Buffer.compare( + control.subarray(payloadOffset + 4, payloadOffset + 6), + Buffer.from([0x08, 0x10]), + ), + ).toBe(0); const payload = Buffer.concat([ control.subarray(payloadOffset, payloadOffset + 4), Buffer.from([0x18, 0x00, 0x10]), @@ -153,7 +156,7 @@ cliTest( }), }), ); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } }, ); diff --git a/tests/acceptance/goBinaryCli.test.ts b/tests/acceptance/goBinaryCli.test.ts index d52a729242..134b8f01b7 100644 --- a/tests/acceptance/goBinaryCli.test.ts +++ b/tests/acceptance/goBinaryCli.test.ts @@ -550,8 +550,11 @@ for (const format of ["elf", "pe", "macho"] as const) { .toString("utf8"), ).toBe(report.build_info.go_version); expect( - fixture.bytes.subarray(module.offset, module.offset + module.bytes), - ).toEqual(Buffer.from(report.build_info.module_bytes_base64, "base64")); + Buffer.compare( + fixture.bytes.subarray(module.offset, module.offset + module.bytes), + Buffer.from(report.build_info.module_bytes_base64, "base64"), + ), + ).toBe(0); }, ); } diff --git a/tests/acceptance/goBinaryPointerMappingsCli.test.ts b/tests/acceptance/goBinaryPointerMappingsCli.test.ts index 6fc239d49a..6fc14c7a57 100644 --- a/tests/acceptance/goBinaryPointerMappingsCli.test.ts +++ b/tests/acceptance/goBinaryPointerMappingsCli.test.ts @@ -56,7 +56,7 @@ cliTest( const response = await call("inspect_go_binary", { path }); expect(response.isError, JSON.stringify(response)).not.toBe(true); expect(response.structuredContent).toEqual(evidence); - expect(await readFile(path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(path), fixture.bytes)).toBe(0); } const damaged = createGoCrossLoadPointerFixture("contents"); diff --git a/tests/acceptance/godotPck.test.ts b/tests/acceptance/godotPck.test.ts index cbe60ab632..1a1a53d4c0 100644 --- a/tests/acceptance/godotPck.test.ts +++ b/tests/acceptance/godotPck.test.ts @@ -343,7 +343,7 @@ cliTest( ); const before = await readFile(bundle); expect((await cli.run({ arguments: args })).exitCode).toBe(1); - expect(await readFile(bundle)).toEqual(before); + expect(Buffer.compare(await readFile(bundle), before)).toBe(0); await unlink(source); const query = join(root, "query.json"); await writeFile( diff --git a/tests/acceptance/harmonyArtifactJourney.test.ts b/tests/acceptance/harmonyArtifactJourney.test.ts index 0370307438..59fa30b5ff 100644 --- a/tests/acceptance/harmonyArtifactJourney.test.ts +++ b/tests/acceptance/harmonyArtifactJourney.test.ts @@ -188,7 +188,7 @@ test("preserves Harmony archive formats through the public CLI/MCP lifecycle", a expect(empty.structuredContent).toMatchObject({ result: { open: false }, }); - expect(await readFile(path)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(path), Buffer.from(bytes))).toBe(0); } } finally { await client.close(); diff --git a/tests/acceptance/il2cppMetadataCli.test.ts b/tests/acceptance/il2cppMetadataCli.test.ts index 866388a812..18d69fbcc3 100644 --- a/tests/acceptance/il2cppMetadataCli.test.ts +++ b/tests/acceptance/il2cppMetadataCli.test.ts @@ -78,7 +78,7 @@ cliTest( }); expect(response.isError).not.toBe(true); expect(parseCliMetadata(response.structuredContent)).toEqual(report); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } for (const [records, declaredCount, declaredBytes] of [ [0, 1, 0], @@ -123,7 +123,7 @@ cliTest( }); expect(response.isError).toBe(true); expect(parseMcpToolError(response).error).toMatchObject(expected); - expect(await readFile(path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(path), fixture.bytes)).toBe(0); } }, 30000, @@ -682,7 +682,7 @@ cliTest( const before = await readFile(bundle); const overwrite = await runFileCli(cli, { arguments: args }); expect(overwrite.exitCode).toBe(1); - expect(await readFile(bundle)).toEqual(before); + expect(Buffer.compare(await readFile(bundle), before)).toBe(0); const saved = await readEvidenceBundle(bundle); if (!saved.ok) throw saved.error; const evidence = saved.value.records[0]; @@ -928,7 +928,7 @@ cliTest( sha256: createHash("sha256").update(bytes).digest("hex"), bytes: bytes.length, }); - expect(await readFile(binaryPath)).toEqual(bytes); + expect(Buffer.compare(await readFile(binaryPath), bytes)).toBe(0); return report; }; const control = await inspect(createIl2cppBinaryFixture()); @@ -1026,7 +1026,7 @@ cliTest( expect(response.isError, JSON.stringify(response)).not.toBe(true); expect(parseCliMetadata(response.structuredContent)).toEqual(report); } - expect(await readFile(binaryPath)).toEqual(bytes); + expect(Buffer.compare(await readFile(binaryPath), bytes)).toBe(0); } await client.ping(); }, diff --git a/tests/acceptance/managedMetadataDelta.test.ts b/tests/acceptance/managedMetadataDelta.test.ts index 77ed826ce1..ef20016df5 100644 --- a/tests/acceptance/managedMetadataDelta.test.ts +++ b/tests/acceptance/managedMetadataDelta.test.ts @@ -127,7 +127,7 @@ cliTest( parseEvidence(response.structuredContent).normalized_result, ).toEqual(evidence.normalized_result); } - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); await rm(path); } const compared = await call("compare_managed_members", { diff --git a/tests/acceptance/managedSelectedScope.test.ts b/tests/acceptance/managedSelectedScope.test.ts index 9bc1172f86..b4d7e2df45 100644 --- a/tests/acceptance/managedSelectedScope.test.ts +++ b/tests/acceptance/managedSelectedScope.test.ts @@ -48,7 +48,7 @@ cliTest( expect(report.coverage.state).toBe("complete"); sources.push(evidence); } - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); await rm(path); const call = await cli.sharedMcpCall!(); for (const [index, right] of sources.slice(1).entries()) { @@ -129,7 +129,7 @@ cliTest( whole.push(evidence); } } - expect(await readFile(path)).toEqual(content); + expect(Buffer.compare(await readFile(path), content)).toBe(0); await rm(path); } const left = managedMemberInspectionSchema.parse( diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index acdcebaf2a..a2eb97524b 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -163,7 +163,7 @@ cliTest( coverage, }); expect(summary.summary.limitations).toContainEqual(limitation); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); await unlink(path); const source = { @@ -419,7 +419,7 @@ cliTest( reason.includes("have released") || reason.includes("has released"), )) expect(summary.summary.limitations).toContain(limitation); - expect(await readFile(path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(path), fixture.bytes)).toBe(0); await unlink(path); await assertRetainedLayoutViews( call, diff --git a/tests/acceptance/nativeDispatchSnapshot.test.ts b/tests/acceptance/nativeDispatchSnapshot.test.ts index a0e6207c83..aa57f1115e 100644 --- a/tests/acceptance/nativeDispatchSnapshot.test.ts +++ b/tests/acceptance/nativeDispatchSnapshot.test.ts @@ -113,8 +113,8 @@ cliTest.skipIf(process.platform !== "darwin")( ).toEqual( savedSnapshot.value.workflow_entries[0]?.execution.analysis_profile, ); - expect(await readFile(first)).toEqual(Buffer.from(bytes)); - expect(await readFile(second)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(first), Buffer.from(bytes))).toBe(0); + expect(Buffer.compare(await readFile(second), Buffer.from(bytes))).toBe(0); }, 30000, ); @@ -170,7 +170,7 @@ cliTest.skipIf(process.platform !== "darwin")( }, }); } - expect(await readFile(target)).toEqual(bytes); + expect(Buffer.compare(await readFile(target), bytes)).toBe(0); await client.ping(); }, 30000, @@ -304,7 +304,7 @@ cliTest.skipIf(process.platform !== "darwin")( }); expect(reobserved.isError, JSON.stringify(reobserved)).not.toBe(true); expect(parseEvidence(reobserved.structuredContent)).toEqual(mcpEvidence); - expect(await readFile(target)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(target), Buffer.from(bytes))).toBe(0); await client.ping(); }, 30000, diff --git a/tests/acceptance/nodeSeaCli.test.ts b/tests/acceptance/nodeSeaCli.test.ts index 4b63a99720..a28623d3e5 100644 --- a/tests/acceptance/nodeSeaCli.test.ts +++ b/tests/acceptance/nodeSeaCli.test.ts @@ -228,7 +228,9 @@ cliTest( expected.map((item) => item.path).sort(), ); for (const { path, body } of expected) - expect(await readFile(join(result.output_root, path))).toEqual(body); + expect( + Buffer.compare(await readFile(join(result.output_root, path)), body), + ).toBe(0); }; const inspected = await cli.run({ diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index 0fdad05fe0..eb27a10e3c 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -289,16 +289,29 @@ cliTest( ); try { expect( - await readFile(join(result.output_root, "pkg", "data.bin")), - ).toEqual(stored); + Buffer.compare( + await readFile(join(result.output_root, "pkg", "data.bin")), + stored, + ), + ).toBe(0); expect( - await readFile(join(result.output_root, "PYZ.pyz", "res", "icon.bin")), - ).toEqual(icon); + Buffer.compare( + await readFile( + join(result.output_root, "PYZ.pyz", "res", "icon.bin"), + ), + icon, + ), + ).toBe(0); expect( - ( - await readFile(join(result.output_root, "PYZ.pyz", "pkg", "mod.pyc")) - ).subarray(16), - ).toEqual(module); + Buffer.compare( + ( + await readFile( + join(result.output_root, "PYZ.pyz", "pkg", "mod.pyc"), + ) + ).subarray(16), + module, + ), + ).toBe(0); } finally { await rm(result.output_root, { recursive: true, force: true }); } @@ -321,9 +334,12 @@ it("extracts legacy and modern CArchive modules with one pyc header through the ); try { for (const module of fixture.modules) { - expect(await readFile(join(extraction.output_root, module.name))).toEqual( - module.bytes, - ); + expect( + Buffer.compare( + await readFile(join(extraction.output_root, module.name)), + module.bytes, + ), + ).toBe(0); expect( extraction.artifacts.find((item) => item.relative_path === module.name), ).toMatchObject({ @@ -465,9 +481,12 @@ it("extracts CArchive members through extract-artifact", async () => { expect( await readFile(join(extracted.output_root, "PYZ.pyz", "app.pyc")), ).toHaveLength(16 + Buffer.from("app-body").length); - expect(await readFile(join(extracted.output_root, "hello"))).toEqual( - Buffer.from("print('hi')"), - ); + expect( + Buffer.compare( + await readFile(join(extracted.output_root, "hello")), + Buffer.from("print('hi')"), + ), + ).toBe(0); expect(await readdir(extracted.output_root)).not.toContain("__pyinstaller__"); expect( extracted.artifacts.some((item) => @@ -564,11 +583,17 @@ it("preserves a PYZ namespace declaration beside a physical CArchive file at the expect.arrayContaining(["PYZ.pyz/ns", "healthy.txt"]), ); expect( - await readFile(join(extracted.output_root, "PYZ.pyz", "ns")), - ).toEqual(Buffer.from("outer data")); - expect(await readFile(join(extracted.output_root, "healthy.txt"))).toEqual( - Buffer.from("healthy sibling"), - ); + Buffer.compare( + await readFile(join(extracted.output_root, "PYZ.pyz", "ns")), + Buffer.from("outer data"), + ), + ).toBe(0); + expect( + Buffer.compare( + await readFile(join(extracted.output_root, "healthy.txt")), + Buffer.from("healthy sibling"), + ), + ).toBe(0); } finally { await rm(extracted.output_root, { recursive: true, force: true }); } diff --git a/tests/acceptance/rustBinaryCli.test.ts b/tests/acceptance/rustBinaryCli.test.ts index 1a4cb4a718..ae409a734d 100644 --- a/tests/acceptance/rustBinaryCli.test.ts +++ b/tests/acceptance/rustBinaryCli.test.ts @@ -586,7 +586,7 @@ cliTest( { stage: "denied", handleId: 0, open: true }, { stage: "released", handleId: 0, fd: -1 }, ]); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); diff --git a/tests/acceptance/rustUnnamedSections.test.ts b/tests/acceptance/rustUnnamedSections.test.ts index f6be212e94..5e50bb2c56 100644 --- a/tests/acceptance/rustUnnamedSections.test.ts +++ b/tests/acceptance/rustUnnamedSections.test.ts @@ -52,7 +52,7 @@ cliTest( const response = await call("inspect_rust_binary", { path }); expect(response.isError, JSON.stringify(response)).not.toBe(true); expect(response.structuredContent).toEqual(evidence); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } await client.ping(); }, diff --git a/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts b/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts index 8ecf960dbb..3aa9a21432 100644 --- a/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts +++ b/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts @@ -166,7 +166,7 @@ for (const scenario of cases) }); if (!("syntax" in scenario)) expect(uninstall.stdout).not.toContain("not valid JSON"); - expect(await readFile(config)).toEqual(original); + expect(Buffer.compare(await readFile(config), original)).toBe(0); expect(await readdir(dirname(config))).toEqual([ scenario.path.split("/").at(-1), ]); diff --git a/tests/acceptance/setup/geminiWorkspaceTrust.test.ts b/tests/acceptance/setup/geminiWorkspaceTrust.test.ts index a1d19c0125..32d4711cff 100644 --- a/tests/acceptance/setup/geminiWorkspaceTrust.test.ts +++ b/tests/acceptance/setup/geminiWorkspaceTrust.test.ts @@ -63,7 +63,9 @@ cliTest( code: "ENOENT", }); expect(await readFile(trust, "utf8")).toBe(text); - expect(await readFile(fixture.user)).toEqual(fixture.originalUser); + expect( + Buffer.compare(await readFile(fixture.user), fixture.originalUser), + ).toBe(0); } const workspaceText = "{ not json"; @@ -92,7 +94,9 @@ cliTest( }); expect(await readFile(fixture.workspace, "utf8")).toBe(workspaceText); expect(await readFile(trust, "utf8")).toBe("{ not json"); - expect(await readFile(fixture.user)).toEqual(fixture.originalUser); + expect( + Buffer.compare(await readFile(fixture.user), fixture.originalUser), + ).toBe(0); }, ); @@ -135,8 +139,12 @@ for (const scenario of [ ]), }, }); - expect(await readFile(fixture.workspace)).toEqual(malformed); - expect(await readFile(fixture.user)).toEqual(fixture.originalUser); + expect(Buffer.compare(await readFile(fixture.workspace), malformed)).toBe( + 0, + ); + expect( + Buffer.compare(await readFile(fixture.user), fixture.originalUser), + ).toBe(0); await expect( access(`${fixture.workspace}.rea.backup`), ).rejects.toMatchObject({ @@ -162,7 +170,9 @@ for (const scenario of [ ]), }, }); - expect(await readFile(fixture.workspace)).toEqual(excluded); + expect( + Buffer.compare(await readFile(fixture.workspace), excluded), + ).toBe(0); } }, ); @@ -208,8 +218,12 @@ cliTest( ]), }, }); - expect(await readFile(fixture.workspace)).toEqual(original); - expect(await readFile(fixture.user)).toEqual(fixture.originalUser); + expect(Buffer.compare(await readFile(fixture.workspace), original)).toBe( + 0, + ); + expect( + Buffer.compare(await readFile(fixture.user), fixture.originalUser), + ).toBe(0); await expect( access(`${fixture.workspace}.rea.backup`), ).rejects.toMatchObject({ diff --git a/tests/acceptance/setup/selectedSkillSetup.test.ts b/tests/acceptance/setup/selectedSkillSetup.test.ts index 8c76e1ac68..9dd3dad5b8 100644 --- a/tests/acceptance/setup/selectedSkillSetup.test.ts +++ b/tests/acceptance/setup/selectedSkillSetup.test.ts @@ -173,12 +173,18 @@ cliTest( }), ]), }); - expect(await readFile(join(selected, "SKILL.md"))).toEqual( - await readFile(join(source, "SKILL.md")), - ); - expect(await readFile(join(managed, "native-and-artifacts.md"))).toEqual( - before, - ); + expect( + Buffer.compare( + await readFile(join(selected, "SKILL.md")), + await readFile(join(source, "SKILL.md")), + ), + ).toBe(0); + expect( + Buffer.compare( + await readFile(join(managed, "native-and-artifacts.md")), + before, + ), + ).toBe(0); expect((await lstat(join(selected, "references"))).isSymbolicLink()).toBe( true, ); @@ -247,9 +253,12 @@ cliTest( })) { if (!entry.isFile()) continue; const path = relative(bundle, join(entry.parentPath, entry.name)); - expect(await readFile(join(selected, path))).toEqual( - await readFile(join(bundle, path)), - ); + expect( + Buffer.compare( + await readFile(join(selected, path)), + await readFile(join(bundle, path)), + ), + ).toBe(0); } }; await assertAligned(); diff --git a/tests/acceptance/setup/unqualifiedDoctorSkill.test.ts b/tests/acceptance/setup/unqualifiedDoctorSkill.test.ts index 36a1a63f21..606ba8c39e 100644 --- a/tests/acceptance/setup/unqualifiedDoctorSkill.test.ts +++ b/tests/acceptance/setup/unqualifiedDoctorSkill.test.ts @@ -42,7 +42,7 @@ cliTest( }, }, }); - expect(await readFile(skill)).toEqual(original); + expect(Buffer.compare(await readFile(skill), original)).toBe(0); const audit = await run(["doctor"]); expect(audit.json).toMatchObject({ @@ -54,7 +54,7 @@ cliTest( }, }, }); - expect(await readFile(skill)).toEqual(original); + expect(Buffer.compare(await readFile(skill), original)).toBe(0); await mkdir(join(home, ".cursor")); const withMarker = await run(["doctor"]); @@ -67,7 +67,7 @@ cliTest( }, }, }); - expect(await readFile(skill)).toEqual(original); + expect(Buffer.compare(await readFile(skill), original)).toBe(0); await writeFile( skill, @@ -102,7 +102,7 @@ cliTest( }, }, }); - expect(await readFile(reference)).toEqual(retained); + expect(Buffer.compare(await readFile(reference), retained)).toBe(0); } const emptyHome = await createTestTempDirectory( @@ -146,6 +146,6 @@ cliTest( healthy: false, identity: { skill: { state: "stale", installed_version: null } }, }); - expect(await readFile(foreignSkill)).toEqual(foreignBytes); + expect(Buffer.compare(await readFile(foreignSkill), foreignBytes)).toBe(0); }, ); diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index 83b129c77e..b3b1c16a05 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -322,7 +322,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( { path, table: "selected", columns: ["ID"] }, ]) expect((await call("inspect_sqlite_database", input)).isError).toBe(true); - expect(await readFile(path)).toEqual(before); + expect(Buffer.compare(await readFile(path), before)).toBe(0); }, ); @@ -632,7 +632,7 @@ cliTest }); expect(response.isError, JSON.stringify(response)).not.toBe(true); expect(response.structuredContent).toEqual(result.json); - expect(await readFile(path)).toEqual(original); + expect(Buffer.compare(await readFile(path), original)).toBe(0); }, ); @@ -1074,7 +1074,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, }, }); - expect(await readFile(path)).toEqual(before); + expect(Buffer.compare(await readFile(path), before)).toBe(0); }, ); @@ -1136,7 +1136,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( expect(response.isError, JSON.stringify(response)).not.toBe(true); expect(response.structuredContent).toEqual(result.json); } - expect(await readFile(path)).toEqual(original); + expect(Buffer.compare(await readFile(path), original)).toBe(0); }, ); @@ -1338,7 +1338,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( const response = await call("inspect_sqlite_database", { path }); expect(response.isError, JSON.stringify(response)).not.toBe(true); expect(response.structuredContent).toEqual(result.json); - expect(await readFile(path)).toEqual(original); + expect(Buffer.compare(await readFile(path), original)).toBe(0); } }, ); diff --git a/tests/acceptance/tools/nativeFunctionMatching.test.ts b/tests/acceptance/tools/nativeFunctionMatching.test.ts index 3afed436ac..8c083945ff 100644 --- a/tests/acceptance/tools/nativeFunctionMatching.test.ts +++ b/tests/acceptance/tools/nativeFunctionMatching.test.ts @@ -162,7 +162,7 @@ const verifyPortableCli = async ( expect(ids.has(dependency.evidence_id)).toBe(true); expect(ids.has(unrelated.evidence_id)).toBe(false); expect(ids.has(evidenceId)).toBe(true); - expect(await readFile(bundlePath)).toEqual(original); + expect(Buffer.compare(await readFile(bundlePath), original)).toBe(0); const incomplete = await cli.run({ arguments: [ "match-native-functions", diff --git a/tests/acceptance/tools/protobufDescriptorFiles.test.ts b/tests/acceptance/tools/protobufDescriptorFiles.test.ts index 80c6354139..6b964a085a 100644 --- a/tests/acceptance/tools/protobufDescriptorFiles.test.ts +++ b/tests/acceptance/tools/protobufDescriptorFiles.test.ts @@ -198,7 +198,9 @@ cliTest( }, }); } - expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + expect( + Buffer.compare(await readFile(descriptor), Buffer.from(bytes)), + ).toBe(0); expect(await readFile(capture, "utf8")).toBe(text); } }, @@ -295,7 +297,9 @@ cliTest( expect(parseEvidence(response.structuredContent).evidence_id).toBe( evidence.evidence_id, ); - expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + expect( + Buffer.compare(await readFile(descriptor), Buffer.from(bytes)), + ).toBe(0); } }, ); diff --git a/tests/acceptance/tools/protobufDescriptorIdentity.test.ts b/tests/acceptance/tools/protobufDescriptorIdentity.test.ts index b19dbf8315..cef90a4968 100644 --- a/tests/acceptance/tools/protobufDescriptorIdentity.test.ts +++ b/tests/acceptance/tools/protobufDescriptorIdentity.test.ts @@ -165,7 +165,9 @@ cliTest( protobuf_coverage: { status: "partial", inspected_records: 0 }, }, }); - expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + expect( + Buffer.compare(await readFile(descriptor), Buffer.from(bytes)), + ).toBe(0); } expect(await readFile(capture, "utf8")).toBe(text); }, @@ -257,6 +259,8 @@ cliTest( expect(parseEvidence(response.structuredContent).evidence_id).toBe( evidence.evidence_id, ); - expect(await readFile(descriptor)).toEqual(Buffer.from(definition)); + expect( + Buffer.compare(await readFile(descriptor), Buffer.from(definition)), + ).toBe(0); }, ); diff --git a/tests/acceptance/tools/protobufRegistryIdentity.test.ts b/tests/acceptance/tools/protobufRegistryIdentity.test.ts index 6c2004f7dc..f6cc0bb65b 100644 --- a/tests/acceptance/tools/protobufRegistryIdentity.test.ts +++ b/tests/acceptance/tools/protobufRegistryIdentity.test.ts @@ -149,7 +149,9 @@ cliTest( parseMcpToolError(response).error, `${scope}-${reverse}`, ).toMatchObject(expected); - expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + expect( + Buffer.compare(await readFile(descriptor), Buffer.from(bytes)), + ).toBe(0); } expect(await readFile(capture, "utf8")).toBe(text); }, @@ -216,6 +218,8 @@ cliTest( { number: 1, schema_name: fieldName, schema_value: "1" }, ]); } - expect(await readFile(descriptor)).toEqual(Buffer.from(bytes)); + expect(Buffer.compare(await readFile(descriptor), Buffer.from(bytes))).toBe( + 0, + ); }, ); diff --git a/tests/boundary/binaryDiagnostics/RecordedCrashStructureBindings.test.ts b/tests/boundary/binaryDiagnostics/RecordedCrashStructureBindings.test.ts index 6db6ada328..38c0dd9489 100644 --- a/tests/boundary/binaryDiagnostics/RecordedCrashStructureBindings.test.ts +++ b/tests/boundary/binaryDiagnostics/RecordedCrashStructureBindings.test.ts @@ -60,7 +60,7 @@ it.each([ report.decoder_diagnostics, ), ).not.toThrow(); - expect(snapshot).toEqual(original); + expect(Buffer.compare(snapshot, original)).toBe(0); expect(report.note_interpretation_completeness).toBe("unknown"); expect(report.live_process_identity).toBe("unknown"); }, @@ -178,6 +178,6 @@ it.each(["/usr/bin/fixture", "../../fixture", "/a/../fixture"])( expect(validateCoreFileMappingAuthority(report, snapshot).ok).toBe( path === "/usr/bin/fixture", ); - expect(snapshot).toEqual(original); + expect(Buffer.compare(snapshot, original)).toBe(0); }, ); diff --git a/tests/boundary/binaryDiagnostics/recordedCrashProvider.test.ts b/tests/boundary/binaryDiagnostics/recordedCrashProvider.test.ts index 46343156b4..f2d3d05db6 100644 --- a/tests/boundary/binaryDiagnostics/recordedCrashProvider.test.ts +++ b/tests/boundary/binaryDiagnostics/recordedCrashProvider.test.ts @@ -159,7 +159,9 @@ it const request = requestSchema.parse( JSON.parse(await readFile(spawn.arguments.at(-1) ?? "", "utf8")), ); - expect(await readFile(request.snapshot_path)).toEqual(bytes); + expect( + Buffer.compare(await readFile(request.snapshot_path), bytes), + ).toBe(0); const payload = structuredClone(value); alterRecordedCrashReply(payload, scenario); const { @@ -201,7 +203,7 @@ it capturedOutput: { stdout: "recorded warning" }, }, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); await expect(access(owned)).rejects.toMatchObject({ code: "ENOENT" }); }, ); @@ -257,7 +259,9 @@ it const request = requestSchema.parse( JSON.parse(await readFile(spawn.arguments.at(-1) ?? "", "utf8")), ); - expect(await readFile(request.snapshot_path)).toEqual(bytes); + expect( + Buffer.compare(await readFile(request.snapshot_path), bytes), + ).toBe(0); if (snapshotPath === "") { snapshotPath = request.snapshot_path; owned = spawn.cwd ?? ""; @@ -341,14 +345,14 @@ it } if (scenario === "cancelled" || scenario === "cancelled-exec") expectDebuggerStopped(debuggerProcess); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); if ( scenario === "cleanup-failure" || scenario === "decoder-cleanup-failure" || scenario === "cancelled-exec-foreign" ) { await expect(access(owned)).resolves.toBeUndefined(); - expect(await readFile(snapshotPath)).toEqual(bytes); + expect(Buffer.compare(await readFile(snapshotPath), bytes)).toBe(0); const refused = await provider.inspect({ path, include_debugger_context: true, @@ -538,7 +542,7 @@ it.runIf(host)( include_debugger_context: true, pwntools_timeout_ms: 1_500, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); if (result.ok) throw new Error("expected debugger timeout"); const projected = projectAnalysisError(result.error); expect(projected.code).toBe("provider_timeout"); diff --git a/tests/boundary/browser/sazWebCapture.test.ts b/tests/boundary/browser/sazWebCapture.test.ts index 1b5fff10c6..084bfa84b0 100644 --- a/tests/boundary/browser/sazWebCapture.test.ts +++ b/tests/boundary/browser/sazWebCapture.test.ts @@ -286,7 +286,7 @@ it("checks trailer credentials before hidden metadata blocks analysis and withho ); expect(opaque.records[0]?.protobuf).toBeUndefined(); expect(opaque.protobuf_coverage?.status).toBe("partial"); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); it("keeps automatic framing applicability unknown when SAZ Content-Type is excluded or conflicting", async () => { @@ -394,7 +394,7 @@ it("keeps automatic framing applicability unknown when SAZ Content-Type is exclu expect(explicit.records[0]?.protobuf).toMatchObject([ { status: "complete", fields: [{ number: 1, value: "1" }] }, ]); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); it("inspects native SAZ through the public CLI and MCP with equal evidence, bytes and archive coordinates", async () => { @@ -470,7 +470,7 @@ it("inspects native SAZ through the public CLI and MCP with equal evidence, byte expect(capture.artifact.sha256).toBe( createHash("sha256").update(bytes).digest("hex"), ); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); const { client, call } = await connectLocalToolsMcp(); const tool = (await client.listTools()).tools.find( @@ -624,7 +624,7 @@ it("keeps CRLF separators inside a body after LF-only HTTP headers", async () => bytes: body.length, sha256: createHash("sha256").update(body).digest("hex"), }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); it("reports zero-byte SAZ HTTP members as null instead of rejecting the archive", async () => { @@ -890,7 +890,7 @@ it.each([ expect(error.details).not.toHaveProperty("partial_observation"); expect(mcpFailure).not.toHaveProperty("normalized_result"); expect(mcpFailure.error.details).not.toHaveProperty("partial_observation"); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } expect(JSON.stringify(result)).not.toContain("malformed-private-input"); }); diff --git a/tests/boundary/cli/artifactCallerSelection.test.ts b/tests/boundary/cli/artifactCallerSelection.test.ts index 72c86395ce..ae5f76bbe8 100644 --- a/tests/boundary/cli/artifactCallerSelection.test.ts +++ b/tests/boundary/cli/artifactCallerSelection.test.ts @@ -62,7 +62,7 @@ cliTest( action: "Evidence file already exists. Choose another output path.", }, }); - expect(await readFile(output)).toEqual(before); + expect(Buffer.compare(await readFile(output), before)).toBe(0); const next = await cli.run({ arguments: [ "inspect-artifact", diff --git a/tests/boundary/cli/commandEvidenceDelivery.test.ts b/tests/boundary/cli/commandEvidenceDelivery.test.ts index 1f04135019..f6bcf935a4 100644 --- a/tests/boundary/cli/commandEvidenceDelivery.test.ts +++ b/tests/boundary/cli/commandEvidenceDelivery.test.ts @@ -199,7 +199,7 @@ it.each(kinds)( detail: "summary", evidence_output: destination, }); - expect(await readFile(destination)).toEqual(original); + expect(Buffer.compare(await readFile(destination), original)).toBe(0); expect(output).toHaveProperty("code"); expect(output).not.toHaveProperty("summary"); expect(output).toHaveProperty("details.partial_observation.evidence_id"); diff --git a/tests/boundary/cli/environmentBuildCli.test.ts b/tests/boundary/cli/environmentBuildCli.test.ts index 206ec83960..3a3de02e6c 100644 --- a/tests/boundary/cli/environmentBuildCli.test.ts +++ b/tests/boundary/cli/environmentBuildCli.test.ts @@ -88,9 +88,13 @@ cliTest( expect(files).toEqual(await listFiles(join(root, "three-step"))); expect(files).toHaveLength(7); for (const file of files) - expect(await readFile(join(root, "built", file)), file).toEqual( - await readFile(join(root, "three-step", file)), - ); + expect( + Buffer.compare( + await readFile(join(root, "built", file)), + await readFile(join(root, "three-step", file)), + ), + file, + ).toBe(0); expect(built.json).toMatchObject({ tool_count: (exported.json as { tool_count: number }).tool_count, files: (exported.json as { files: unknown[] }).files, diff --git a/tests/boundary/cli/environmentExportCli.test.ts b/tests/boundary/cli/environmentExportCli.test.ts index 97fd6ba47d..23395c9db1 100644 --- a/tests/boundary/cli/environmentExportCli.test.ts +++ b/tests/boundary/cli/environmentExportCli.test.ts @@ -116,11 +116,17 @@ cliTest( .sort(); expect(written).toEqual(BUNDLE_FILES); expect( - await readFile(join(directory, "envs/grafana-example/manifest.json")), - ).toEqual(await readFile(files.manifestPath)); + Buffer.compare( + await readFile(join(directory, "envs/grafana-example/manifest.json")), + await readFile(files.manifestPath), + ), + ).toBe(0); expect( - await readFile(join(directory, "envs/grafana-example/evidence.json")), - ).toEqual(await readFile(files.capturePath)); + Buffer.compare( + await readFile(join(directory, "envs/grafana-example/evidence.json")), + await readFile(files.capturePath), + ), + ).toBe(0); const dockerfile = await readFile( join(directory, "envs/grafana-example/Dockerfile"), "utf8", @@ -253,8 +259,11 @@ cliTest.skipIf(process.platform === "win32")( ).toBeGreaterThan(0); } expect( - await readFile(join(directory, "envs/grafana-example/evidence.json")), - ).toEqual(await readFile(files.capturePath)); + Buffer.compare( + await readFile(join(directory, "envs/grafana-example/evidence.json")), + await readFile(files.capturePath), + ), + ).toBe(0); }, ); @@ -290,10 +299,13 @@ cliTest.skipIf(process.platform === "win32")( package_source: { kind: "tarball", path: packagePath }, }); expect( - await readFile( - join(physical, "bundle/envs/grafana-example/rea-agents.tgz"), + Buffer.compare( + await readFile( + join(physical, "bundle/envs/grafana-example/rea-agents.tgz"), + ), + bytes, ), - ).toEqual(bytes); + ).toBe(0); await expect( readFile(join(root, "bundle/README.md")), ).rejects.toMatchObject({ code: "ENOENT" }); diff --git a/tests/boundary/cli/environmentExportInputsCli.test.ts b/tests/boundary/cli/environmentExportInputsCli.test.ts index 17ebf3cc53..cf1e986ac9 100644 --- a/tests/boundary/cli/environmentExportInputsCli.test.ts +++ b/tests/boundary/cli/environmentExportInputsCli.test.ts @@ -49,9 +49,12 @@ for (const role of ["manifest", "evidence"] as const) { timeoutMs: 30_000, }); expect(await readFile(marker, "utf8")).toBe("read"); - expect(await readFile(input)).toEqual( - Buffer.concat([before, Buffer.from(" ")]), - ); + expect( + Buffer.compare( + await readFile(input), + Buffer.concat([before, Buffer.from(" ")]), + ), + ).toBe(0); expect(result.exitCode, result.stderr).toBe(1); expect(result.json).toMatchObject({ code: "invalid_request", diff --git a/tests/boundary/cli/idaDoctorRegistration.test.ts b/tests/boundary/cli/idaDoctorRegistration.test.ts index e7965c8663..1424087761 100644 --- a/tests/boundary/cli/idaDoctorRegistration.test.ts +++ b/tests/boundary/cli/idaDoctorRegistration.test.ts @@ -82,7 +82,9 @@ cliTest.skipIf(process.platform !== "darwin").for([true, false])( }), ]), }); - expect(await readFile(registration)).toEqual(registrationBytes); + expect( + Buffer.compare(await readFile(registration), registrationBytes), + ).toBe(0); expect(await readdir(physical)).toEqual(before); if (usable) { expect(await readdir(parent)).toEqual([]); diff --git a/tests/boundary/cli/il2cppSharedSuffixNames.test.ts b/tests/boundary/cli/il2cppSharedSuffixNames.test.ts index f8166b4abb..21b761268c 100644 --- a/tests/boundary/cli/il2cppSharedSuffixNames.test.ts +++ b/tests/boundary/cli/il2cppSharedSuffixNames.test.ts @@ -59,7 +59,7 @@ cliTest( expect(mcpOutput.isError, JSON.stringify(mcpOutput)).not.toBe(true); const mcpResult = parseResult(mcpOutput.structuredContent); expect(mcpResult).toEqual(cliResult); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } }, ); diff --git a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts index d730ee2326..5d3b15bd69 100644 --- a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts +++ b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts @@ -117,7 +117,7 @@ cliTest( ), ).toEqual(result); } - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } }, ); diff --git a/tests/boundary/cli/managedMemberSummaryCli.test.ts b/tests/boundary/cli/managedMemberSummaryCli.test.ts index 82cfbecbe5..7756fca24c 100644 --- a/tests/boundary/cli/managedMemberSummaryCli.test.ts +++ b/tests/boundary/cli/managedMemberSummaryCli.test.ts @@ -346,7 +346,7 @@ cliTest( expect(result.json).not.toHaveProperty("evidence_id"); expect(result.json).not.toHaveProperty("normalized_result"); } - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); expect(await readdir(root)).toEqual(["fixture.dll"]); }, ); diff --git a/tests/boundary/cli/nativePlistCli.test.ts b/tests/boundary/cli/nativePlistCli.test.ts index 192f2337ab..3b10587291 100644 --- a/tests/boundary/cli/nativePlistCli.test.ts +++ b/tests/boundary/cli/nativePlistCli.test.ts @@ -113,7 +113,7 @@ describe.skipIf(process.platform !== "darwin")( ]), }, }); - expect(await readFile(path)).toEqual(source); + expect(Buffer.compare(await readFile(path), source)).toBe(0); }, ); diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.ts index a2451a31ac..f33bd285c2 100644 --- a/tests/boundary/cli/referenceSourceUtf8.test.ts +++ b/tests/boundary/cli/referenceSourceUtf8.test.ts @@ -185,8 +185,11 @@ cliTest.skipIf(process.platform === "win32")( }), ); expect( - await readlink(join(root, "alias.ts"), { encoding: "buffer" }), - ).toEqual(rawTarget); + Buffer.compare( + await readlink(join(root, "alias.ts"), { encoding: "buffer" }), + rawTarget, + ), + ).toBe(0); }, ); diff --git a/tests/boundary/evm/evmoleInterfaceProvider.test.ts b/tests/boundary/evm/evmoleInterfaceProvider.test.ts index 70867e67ef..6191079291 100644 --- a/tests/boundary/evm/evmoleInterfaceProvider.test.ts +++ b/tests/boundary/evm/evmoleInterfaceProvider.test.ts @@ -894,7 +894,7 @@ it expect(evidence.subject?.digest.sha256).toBe( createHash("sha256").update(source).digest("hex"), ); - expect(await readFile(path)).toEqual(source); + expect(Buffer.compare(await readFile(path), source)).toBe(0); expect(evidence).toMatchObject({ operation: "inspect_evm_interface", parameters: { path, encoding: "hex" }, diff --git a/tests/boundary/filesystem/appleDispatchMetadataStrings.test.ts b/tests/boundary/filesystem/appleDispatchMetadataStrings.test.ts index da230322c0..e22b794033 100644 --- a/tests/boundary/filesystem/appleDispatchMetadataStrings.test.ts +++ b/tests/boundary/filesystem/appleDispatchMetadataStrings.test.ts @@ -71,7 +71,7 @@ it.each([ ({ facet }) => facet === "objc_properties_categories", ), ).toMatchObject({ status: "complete", reason: null }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); @@ -105,6 +105,6 @@ it.each([ status: "partial", reason: expect.stringContaining("property_decode_failed"), }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }, ); diff --git a/tests/boundary/filesystem/artifactLiteralBackslash.test.ts b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts index b56bdb6327..931e0a9c34 100644 --- a/tests/boundary/filesystem/artifactLiteralBackslash.test.ts +++ b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts @@ -117,7 +117,9 @@ it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( [...new Set(NAMES.map((name) => name.split("/")[0]))].sort(), ); for (const name of NAMES) - expect(await readFile(join(output, name))).toEqual(CONTENT); + expect(Buffer.compare(await readFile(join(output, name)), CONTENT)).toBe( + 0, + ); }, ); diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index 8ad141f4d0..232ca521d7 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -83,7 +83,7 @@ for (const zip64 of [false, true]) { expect(extraction.artifacts).toEqual([]); expect(extraction.containment_verified).toBe(true); expect(await readdir(extraction.output_root)).toEqual([]); - expect(await readFile(archive)).toEqual(bytes); + expect(Buffer.compare(await readFile(archive), bytes)).toBe(0); }, ); } diff --git a/tests/boundary/filesystem/asarHeaderAdmission.test.ts b/tests/boundary/filesystem/asarHeaderAdmission.test.ts index f7fb0d30a0..20ead66f6d 100644 --- a/tests/boundary/filesystem/asarHeaderAdmission.test.ts +++ b/tests/boundary/filesystem/asarHeaderAdmission.test.ts @@ -72,7 +72,7 @@ it("admits a header produced by the upstream ASAR serializer", async () => { headerBytes: original.readUInt32LE(4), jsonBytes: original.readUInt32LE(12), }); - expect(await readFile(archive)).toEqual(original); + expect(Buffer.compare(await readFile(archive), original)).toBe(0); }); it("rejects every truncated prefix of a real serialized header", async () => { diff --git a/tests/boundary/filesystem/asarListingAdmission.test.ts b/tests/boundary/filesystem/asarListingAdmission.test.ts index 9762adff9f..349e3b0343 100644 --- a/tests/boundary/filesystem/asarListingAdmission.test.ts +++ b/tests/boundary/filesystem/asarListingAdmission.test.ts @@ -47,7 +47,7 @@ it("rejects path expansion before publishing any inventory, preserving the input reason: "limit", message: "ASAR expanded listing exceeds its 64 MiB representation budget", }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); it("reads deeply nested packed metadata without recursive listing or member lookup", async () => { @@ -59,7 +59,12 @@ it("reads deeply nested packed metadata without recursive listing or member look count++; if (entry.kind === "file") { expect(entry.path).toBe(`${"d/".repeat(3500)}f0`); - expect(await buffer(await reader.open(entry))).toEqual(Buffer.alloc(0)); + expect( + Buffer.compare( + await buffer(await reader.open(entry)), + Buffer.alloc(0), + ), + ).toBe(0); } } expect(count).toBe(3501); @@ -123,9 +128,12 @@ it("preserves POSIX literal backslashes and rejects Windows component separators const entries = await list(); expect(entries).toHaveLength(1); expect(entries[0]?.path).toBe("a\\b"); - expect(await buffer(await reader.open(entries[0]!))).toEqual( - Buffer.alloc(0), - ); + expect( + Buffer.compare( + await buffer(await reader.open(entries[0]!)), + Buffer.alloc(0), + ), + ).toBe(0); } } finally { await reader.close(); diff --git a/tests/boundary/filesystem/boundaryLintConfiguration.test.ts b/tests/boundary/filesystem/boundaryLintConfiguration.test.ts new file mode 100644 index 0000000000..133b13fe31 --- /dev/null +++ b/tests/boundary/filesystem/boundaryLintConfiguration.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const repository = fileURLToPath(new URL("../../../", import.meta.url)); +const oxlint = join(repository, "node_modules/oxlint/bin/oxlint"); +const config = join(repository, ".oxlintrc.json"); +const reportSchema = z.object({ + diagnostics: z.array( + z.object({ code: z.string().optional(), severity: z.string() }), + ), +}); + +const lint = (path: string) => { + const result = spawnSync( + process.execPath, + [oxlint, "--threads=1", "--config", config, "--format", "json", path], + { cwd: repository, encoding: "utf8", timeout: 15_000 }, + ); + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + const report = reportSchema.parse(JSON.parse(result.stdout)); + return { status: result.status, diagnostics: report.diagnostics }; +}; + +it("enforces boundary and stock correctness rules through the repository config", async () => { + const root = await createTestTempDirectory("rea-boundary-lint-"); + const path = join(root, "boundary-probe.ts"); + const imports = [ + 'import { readFile } from "node:fs/promises";', + 'import { expect } from "vitest";', + 'import { AnalysisCancelledError } from "./analysisErrorCore.js";', + ].join("\n"); + await writeFile( + path, + `${imports}\nasync function inspect(path: string, signal: AbortSignal) { + const before = await readFile(path); + expect(await readFile(path)).toEqual(before); + if (signal.aborted) throw new AnalysisCancelledError("inspect"); + new Promise(async resolve => resolve(1)); + const optional = undefined as { value: number } | undefined; + (optional?.value).toString(); + return; + console.log("unreachable"); + }`, + ); + const failing = lint(path); + expect(failing.status).toBe(1); + expect( + failing.diagnostics + .filter(({ severity }) => severity === "error") + .map(({ code, severity }) => ({ code: code ?? "", severity })) + .sort((left, right) => left.code.localeCompare(right.code)), + ).toEqual([ + { code: "eslint(no-async-promise-executor)", severity: "error" }, + { code: "eslint(no-unreachable)", severity: "error" }, + { code: "eslint(no-unsafe-optional-chaining)", severity: "error" }, + { code: "rea(cancellation-cause)", severity: "error" }, + { code: "rea(no-buffer-deep-equality)", severity: "error" }, + ]); + + await writeFile( + path, + `${imports}\nasync function inspect(path: string, signal: AbortSignal) { + const before = await readFile(path); + expect((await readFile(path)).equals(before)).toBe(true); + if (signal.aborted) + throw new AnalysisCancelledError("inspect", { cause: signal.reason }); + }`, + ); + const corrected = lint(path); + expect(corrected.status).toBe(0); + expect( + corrected.diagnostics.filter(({ code }) => code?.startsWith("rea(")), + ).toEqual([]); +}); diff --git a/tests/boundary/filesystem/boundedFileBytes.test.ts b/tests/boundary/filesystem/boundedFileBytes.test.ts index faa832de19..8a753eb2b2 100644 --- a/tests/boundary/filesystem/boundedFileBytes.test.ts +++ b/tests/boundary/filesystem/boundedFileBytes.test.ts @@ -16,7 +16,10 @@ describe("bounded file byte reads", () => { await writeFile(path, content); const handle = await open(path, "r"); try { - expect(await readBoundedFileBytes(handle, limit)).toEqual(content); + const actual = await readBoundedFileBytes(handle, limit); + if (actual === undefined) + throw new Error("Expected complete file bytes"); + expect(actual.equals(content)).toBe(true); } finally { await handle.close(); } diff --git a/tests/boundary/filesystem/bufferDeepEqualityLint.test.ts b/tests/boundary/filesystem/bufferDeepEqualityLint.test.ts new file mode 100644 index 0000000000..38c433b070 --- /dev/null +++ b/tests/boundary/filesystem/bufferDeepEqualityLint.test.ts @@ -0,0 +1,137 @@ +import { RuleTester } from "oxlint/plugins-dev"; +import { describe, it } from "vitest"; + +import noBufferDeepEquality from "../../../scripts/lint/no-buffer-deep-equality.mjs"; + +RuleTester.describe = describe; +RuleTester.it = it; + +new RuleTester({ + languageOptions: { sourceType: "module", env: { node: true } }, +}).run("no-buffer-deep-equality", noBufferDeepEquality, { + valid: [ + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(await readFile(path, "utf8")).toEqual(text);`, + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(await readFile(path, { encoding: "utf8" })).toEqual(text);`, + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(await readFile(path, options)).toEqual(value);`, + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + const pending = readFile(path); + expect(pending).toEqual(promise);`, + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(readFile(path, "utf8")).resolves.toEqual(text);`, + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect((await readFile(path)).equals(before)).toBe(true);`, + `import { expect } from "vitest"; + const Buffer = { from: () => ({}) }; + expect(Buffer.from("x")).toEqual(value);`, + `import { expect } from "vitest"; + function check(Buffer) { expect(Buffer.alloc(4)).toEqual(value); }`, + `import { expect } from "vitest"; + function check(Buffer) { + const Bytes = Buffer; + expect(Bytes.from("x")).toEqual(value); + }`, + `import { readFile } from "node:fs/promises"; + function check(expect) { expect(readFile(path)).toEqual(value); }`, + `import { expect } from "vitest"; + function check(readFile) { expect(readFile(path)).toEqual(value); }`, + `import { expect } from "other-assertions"; + expect(Buffer.from("x")).toEqual(value);`, + `expect(Buffer.from("x")).toEqual(value);`, + `import { readFile } from "other-fs"; + import { expect } from "vitest"; + expect(readFile(path)).toEqual(value);`, + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(await readFile(...args)).toEqual(value);`, + `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + const data = await readFile(path, "utf8"); + expect(data).toEqual(text);`, + `import { expect } from "vitest"; + expect([1, 2]).toEqual([1, 2]);`, + `import { expect } from "vitest"; + let bytes = Buffer.from("x"); + expect(bytes).toEqual(value);`, + ], + invalid: [ + { + // Exact pre-fix ELF test shape: readFile without an encoding returns Buffer. + code: `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + const before = await readFile(path); + expect(await readFile(path)).toEqual(before);`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { readFileSync as read } from "node:fs"; + import { expect } from "vitest"; + const before = read(path); + expect(value).toStrictEqual(before);`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { readFile as sourceRead } from "node:fs/promises"; + import { expect } from "vitest"; + const read = sourceRead; + const bytes = await read(path); + expect(bytes).not.toEqual(value);`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import * as fs from "node:fs/promises"; + import { expect } from "vitest"; + expect(await fs.readFile(path)).toEqual(value);`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { Buffer as NodeBuffer } from "node:buffer"; + import { expect } from "vitest"; + expect(value).toEqual(NodeBuffer.from("x"));`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { expect } from "vitest"; + const Bytes = Buffer; + const bytes = Bytes.allocUnsafe(16); + expect(bytes).toEqual(value);`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(await readFile(path), "bytes unchanged").toEqual(before);`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(readFile(path)).resolves.toEqual(Buffer.from("x"));`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { readFile } from "node:fs/promises"; + import { expect } from "vitest"; + expect(readFile(path)).resolves.not.toEqual(value);`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { expect } from "vitest"; + expect(Promise.reject(Buffer.from("x"))).rejects.toStrictEqual(Buffer.from("x"));`, + errors: [{ messageId: "nativeBytes" }], + }, + { + code: `import { expect as check } from "vitest"; + check(Buffer.alloc(4)).toEqual(value);`, + errors: [{ messageId: "nativeBytes" }], + }, + ], +}); diff --git a/tests/boundary/filesystem/cancellationCauseLint.test.ts b/tests/boundary/filesystem/cancellationCauseLint.test.ts new file mode 100644 index 0000000000..b88562a149 --- /dev/null +++ b/tests/boundary/filesystem/cancellationCauseLint.test.ts @@ -0,0 +1,135 @@ +import { RuleTester } from "oxlint/plugins-dev"; +import { describe, it } from "vitest"; + +import cancellationCauseRule from "../../../scripts/lint/cancellation-cause.mjs"; + +RuleTester.describe = describe; +RuleTester.it = it; + +const core = + 'import { AnalysisCancelledError } from "./domain/analysisErrorCore.js";'; +const options = + 'import type { ExecutionOptions } from "./application/AnalysisProvider.js";'; + +new RuleTester({ + languageOptions: { parserOptions: { lang: "ts" }, sourceType: "module" }, +}).run("cancellation-cause", cancellationCauseRule, { + valid: [ + // The corrected WABT pre-abort branch retains even null or object reasons. + `${core}\n${options}\nfunction run(options?: ExecutionOptions) { + if (options?.signal?.aborted) + throw new AnalysisCancelledError("inspect", { + cause: options.signal.reason, + }); + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted === true) + return err(new AnalysisCancelledError("inspect", { cause: signal.reason })); + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) + throw new AnalysisCancelledError("inspect", { 'cause': signal.reason }); + }`, + `${core}\nfunction close() { + if (this.closed) throw new AnalysisCancelledError("inspect"); + }`, + `${core}\nfunction run(signal?: AbortSignal, other?: boolean) { + if (signal?.aborted || other) + throw new AnalysisCancelledError("inspect"); + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) + return () => new AnalysisCancelledError("inspect"); + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) + throw new AnalysisCancelledError("inspect", { ...forwardedContext }); + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) + throw new AnalysisCancelledError("inspect", forwardedContext); + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) + throw new AnalysisCancelledError("inspect", { [forwardedKey]: value }); + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) { + const AnalysisCancelledError = Error; + throw new AnalysisCancelledError("inspect"); + } + }`, + `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) throw new Error("cancelled"); + }`, + // Names alone are insufficient: these are unrelated local structures. + `${core}\ntype AbortSignal = { aborted: boolean }; + function run(signal: AbortSignal) { + if (signal.aborted) throw new AnalysisCancelledError("inspect"); + }`, + `${core}\ntype ExecutionOptions = { signal: { aborted: boolean } }; + function run(options: ExecutionOptions) { + if (options.signal.aborted) + throw new AnalysisCancelledError("inspect"); + }`, + `${core}\nfunction run(signal?: unknown) { + if (signal?.aborted) throw new AnalysisCancelledError("inspect"); + }`, + ], + invalid: [ + { + code: `${core}\n${options}\nfunction run(options?: ExecutionOptions) { + if (options?.signal?.aborted) + throw new AnalysisCancelledError("inspect"); + }`, + errors: [{ messageId: "missingCause" }], + }, + { + code: `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted === true) + return err(new AnalysisCancelledError("inspect")); + }`, + errors: [{ messageId: "missingCause" }], + }, + { + code: `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted && ready) + throw new AnalysisCancelledError("inspect", { diagnostics }); + }`, + errors: [{ messageId: "missingCause" }], + }, + { + code: `import { AnalysisCancelledError as Cancelled } from + "./domain/analysisErrorCore.js"; + function run(signal?: AbortSignal) { + if (signal?.aborted) throw new Cancelled("inspect"); + }`, + errors: [{ messageId: "missingCause" }], + }, + { + code: `${core}\n${options}\nfunction run(options?: ExecutionOptions) { + const signal = options?.signal; + if (signal?.aborted) throw new AnalysisCancelledError("inspect"); + }`, + errors: [{ messageId: "missingCause" }], + }, + { + code: `${core}\nimport type { ExecutionOptions as Options } from + "./application/AnalysisProvider.js"; + function run(options?: Options) { + if (options?.signal?.aborted) + return err(new AnalysisCancelledError("inspect")); + }`, + errors: [{ messageId: "missingCause" }], + }, + { + code: `${core}\nfunction run(signal?: AbortSignal) { + if (signal?.aborted) { + const failure = new AnalysisCancelledError("inspect"); + return err(failure); + } + }`, + errors: [{ messageId: "missingCause" }], + }, + ], +}); diff --git a/tests/boundary/filesystem/ciPortableBuildCache.test.ts b/tests/boundary/filesystem/ciPortableBuildCache.test.ts index c059818b61..ab5933e879 100644 --- a/tests/boundary/filesystem/ciPortableBuildCache.test.ts +++ b/tests/boundary/filesystem/ciPortableBuildCache.test.ts @@ -488,7 +488,9 @@ it.skipIf(process.platform === "win32").each([ await rm(join(root, "out"), { recursive: true }); const restored = execute("npm run --silent test:prepare"); expect(restored).toContain(`Cached: ${hits} cached, 2 total`); - expect(await readFile(join(root, "out/build.bin"))).toEqual(artifact); + expect( + Buffer.compare(await readFile(join(root, "out/build.bin")), artifact), + ).toBe(0); await expect(readFile(join(root, "out/seed.bin"))).rejects.toMatchObject({ code: "ENOENT", }); diff --git a/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts b/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts index 1c64d76348..494565b587 100644 --- a/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts +++ b/tests/boundary/filesystem/claudeDesktopClientConfiguration.test.ts @@ -484,7 +484,7 @@ describe("MSIX Claude Desktop registration lifecycle", () => { throw new Error("Expected a blocked update"); expect(failedConfigurationMessage(result)).toContain(privateConfig); expect(failedConfigurationMessage(result)).toContain("Repair access"); - expect(await readFile(appDataConfig)).toEqual(original); + expect(Buffer.compare(await readFile(appDataConfig), original)).toBe(0); expect(await readdir(dirname(appDataConfig))).toEqual([CONFIG]); } finally { await chmod(privateDirectory, mode); diff --git a/tests/boundary/filesystem/clientConfigurationBom.test.ts b/tests/boundary/filesystem/clientConfigurationBom.test.ts index 9eea4cb41e..c10c76741a 100644 --- a/tests/boundary/filesystem/clientConfigurationBom.test.ts +++ b/tests/boundary/filesystem/clientConfigurationBom.test.ts @@ -96,10 +96,18 @@ describe.each(["vscode", "copilot_cli", "opencode", "commandcode"])( status: "unchanged", }, ); - expect(await readFile(client.configPath)).toEqual(Buffer.from(updated)); - expect(await readFile(`${client.configPath}.rea.backup`)).toEqual( - Buffer.from(original), - ); + expect( + Buffer.compare( + await readFile(client.configPath), + Buffer.from(updated), + ), + ).toBe(0); + expect( + Buffer.compare( + await readFile(`${client.configPath}.rea.backup`), + Buffer.from(original), + ), + ).toBe(0); }, ); }, @@ -125,7 +133,9 @@ it.each(['\uFEFF{"servers": {"rea": @}}', '\uFEFF{"setting":\uFEFFtrue}'])( expect( await configureClientConfiguration(client, {}, ["rea", "mcp"]), ).toEqual({ status: "failed", reason: "readback" }); - expect(await readFile(configPath)).toEqual(Buffer.from(original)); + expect( + Buffer.compare(await readFile(configPath), Buffer.from(original)), + ).toBe(0); expect(await readdir(home)).toEqual(["mcp.json"]); }, ); @@ -171,9 +181,12 @@ it("preserves BOM-prefixed JSONC values and comments through update and removal" expect(parseClientConfiguration(removed, client.format).document).toEqual( parseClientConfiguration(original, client.format).document, ); - expect(await readFile(`${client.configPath}.rea.backup`)).toEqual( - Buffer.from(original), - ); + expect( + Buffer.compare( + await readFile(`${client.configPath}.rea.backup`), + Buffer.from(original), + ), + ).toBe(0); }); it.each(["", "\n \n", "\uFEFF"])( diff --git a/tests/boundary/filesystem/clientConfigurationTransactions.test.ts b/tests/boundary/filesystem/clientConfigurationTransactions.test.ts index a22ce751fc..f00eaf6fb2 100644 --- a/tests/boundary/filesystem/clientConfigurationTransactions.test.ts +++ b/tests/boundary/filesystem/clientConfigurationTransactions.test.ts @@ -205,7 +205,9 @@ it("accepts a literal UTF-8 replacement character without changing its value", a ["rea", "mcp"], ); expect(result.status).toBe("configured"); - expect(await readFile(`${configPath}.rea.backup`)).toEqual(original); + expect( + Buffer.compare(await readFile(`${configPath}.rea.backup`), original), + ).toBe(0); expect(JSON.parse(await readFile(configPath, "utf8")).label).toBe("\uFFFD"); }); diff --git a/tests/boundary/filesystem/dosComTarget.test.ts b/tests/boundary/filesystem/dosComTarget.test.ts index 5968c769c6..24bc64fded 100644 --- a/tests/boundary/filesystem/dosComTarget.test.ts +++ b/tests/boundary/filesystem/dosComTarget.test.ts @@ -27,7 +27,7 @@ describe("DOS COM admission", () => { sha256: createHash("sha256").update(bytes).digest("hex"), }, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); const detected = await parseBinaryTarget(path); expect(detected.ok && detected.value.format).not.toBe("dos-com"); }); @@ -59,7 +59,7 @@ describe("DOS COM admission", () => { architecture: "x86", sha256: createHash("sha256").update(bytes).digest("hex"), }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); it("rejects contradictory target kind", async () => { const directory = await createTestTempDirectory("rea-com-target-"); diff --git a/tests/boundary/filesystem/fileHandleChunks.test.ts b/tests/boundary/filesystem/fileHandleChunks.test.ts index 67c26e290d..4c823d3d02 100644 --- a/tests/boundary/filesystem/fileHandleChunks.test.ts +++ b/tests/boundary/filesystem/fileHandleChunks.test.ts @@ -84,7 +84,7 @@ it("removes cancellation observation after normal stream completion", async () = buffer: Buffer.alloc(0), })) as unknown as FileHandle["read"]; const source = readFileHandleChunks({ read }, { signal: controller.signal }); - expect(await buffer(source)).toEqual(Buffer.alloc(0)); + expect(Buffer.compare(await buffer(source), Buffer.alloc(0))).toBe(0); expect(remove).toHaveBeenCalledWith("abort", expect.any(Function)); controller.abort(null); expect(source.errored).toBeNull(); @@ -111,9 +111,12 @@ it.each(["range", "eof"])( }) as FileHandle["read"]; const source = readFileHandleChunks({ read }, { start: 5, end: 7 }); expect(source.readCoverage()).toEqual({ kind: "incomplete", bytes: 0n }); - expect(await buffer(source)).toEqual( - Buffer.alloc(boundary === "range" ? 3 : 1, 0x61), - ); + expect( + Buffer.compare( + await buffer(source), + Buffer.alloc(boundary === "range" ? 3 : 1, 0x61), + ), + ).toBe(0); const coverage = source.readCoverage(); expect(coverage).toEqual({ kind: "complete", diff --git a/tests/boundary/filesystem/ghidraProfileArtifactOwnership.test.ts b/tests/boundary/filesystem/ghidraProfileArtifactOwnership.test.ts index 7d847495d2..98874971af 100644 --- a/tests/boundary/filesystem/ghidraProfileArtifactOwnership.test.ts +++ b/tests/boundary/filesystem/ghidraProfileArtifactOwnership.test.ts @@ -353,7 +353,7 @@ it.each(["seed", "extension"] as const)( cleanupIncomplete: true, cleanupResources: [output], }); - expect(await readFile(output)).toEqual(bytes); + expect(Buffer.compare(await readFile(output), bytes)).toBe(0); allowClose = true; await resources.retryCleanup(); expect(error).toMatchObject({ cleanupIncomplete: false }); diff --git a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts index 9531fb743f..14efe4a46f 100644 --- a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts +++ b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts @@ -110,7 +110,7 @@ it("keeps exact integers when two object-table entries share one record", async expect(result.limitations).not.toContainEqual( expect.stringMatching(/metadata could not be completely observed/u), ); - expect(await readFile(fixture.path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(fixture.path), fixture.bytes)).toBe(0); }); cliTest( diff --git a/tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts b/tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts index 8160b5b841..eaee5fbc35 100644 --- a/tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts +++ b/tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts @@ -186,7 +186,7 @@ cliTest( message: expect.stringContaining("9007199254740992"), }); const after = await readFile(path); - expect(after).toEqual(bytes); + expect(Buffer.compare(after, bytes)).toBe(0); expect(createHash("sha256").update(after).digest("hex")).toBe(originalHash); }, ); diff --git a/tests/boundary/filesystem/rawBinaryTarget.test.ts b/tests/boundary/filesystem/rawBinaryTarget.test.ts index 2757f05073..72e2acbc88 100644 --- a/tests/boundary/filesystem/rawBinaryTarget.test.ts +++ b/tests/boundary/filesystem/rawBinaryTarget.test.ts @@ -69,7 +69,7 @@ describe("raw-binary admission", () => { sha256: createHash("sha256").update(bytes).digest("hex"), }, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); const detected = await parseBinaryTarget(path); expect(detected.ok && detected.value.format).not.toBe("raw-binary"); }); diff --git a/tests/boundary/filesystem/regularFileCancellation.test.ts b/tests/boundary/filesystem/regularFileCancellation.test.ts index 4fb159d10b..82a639c63d 100644 --- a/tests/boundary/filesystem/regularFileCancellation.test.ts +++ b/tests/boundary/filesystem/regularFileCancellation.test.ts @@ -434,7 +434,7 @@ it.each( expect(admitted?.fd).toBe(-1); expect(resources?.hasPending ?? false).toBe(false); await resources?.retryCleanup(); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } finally { vi.restoreAllMocks(); await resources?.close(); diff --git a/tests/boundary/filesystem/safeOutputTreePublication.test.ts b/tests/boundary/filesystem/safeOutputTreePublication.test.ts index 127619ecef..dbe9c22500 100644 --- a/tests/boundary/filesystem/safeOutputTreePublication.test.ts +++ b/tests/boundary/filesystem/safeOutputTreePublication.test.ts @@ -68,9 +68,12 @@ it.skipIf(process.platform === "win32")( output, dirname(output), ]); - expect(await readFile(join(output, "a", "b", "payload.txt"))).toEqual( - bytes, - ); + expect( + Buffer.compare( + await readFile(join(output, "a", "b", "payload.txt")), + bytes, + ), + ).toBe(0); } } finally { observation.refuseSync = undefined; diff --git a/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts b/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts index 9f819d53e8..839aa427da 100644 --- a/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts +++ b/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts @@ -130,7 +130,7 @@ it("retains invalid UTF-8 diagnostics and the uninstall read owner when close fa status: "failed", detail: expect.stringContaining("not valid UTF-8"), }); - expect(await readFile(path)).toEqual(original); + expect(Buffer.compare(await readFile(path), original)).toBe(0); expect(closeAttempts).toHaveLength(3); expect(closeAttempts.every((handle) => handle === closeAttempts[0])).toBe( true, @@ -280,7 +280,9 @@ describe("setup client configuration cleanup ownership", () => { expect(configure).not.toHaveBeenCalled(); expect(closeAttempts).toHaveLength(2); expect(closeAttempts[0]).toBe(closeAttempts[1]); - expect(await readFile(client.configPath)).toEqual(original); + expect( + Buffer.compare(await readFile(client.configPath), original), + ).toBe(0); await expectCloseReclaimsHandle(host, closeAttempts, release); expect(failure.cleanupIncomplete).toBe(false); diff --git a/tests/boundary/filesystem/setupSkill.test.ts b/tests/boundary/filesystem/setupSkill.test.ts index dbeb443837..47e7953599 100644 --- a/tests/boundary/filesystem/setupSkill.test.ts +++ b/tests/boundary/filesystem/setupSkill.test.ts @@ -152,8 +152,8 @@ describe("skill recovery byte preservation", () => { }); if (result.ok) throw new Error("Expected the injected write failure"); expect(result.error.recoveryBackupPaths).toContain(recovery); - expect(await readFile(recovery)).toEqual(original); - expect(await readFile(manifest)).toEqual(original); + expect(Buffer.compare(await readFile(recovery), original)).toBe(0); + expect(Buffer.compare(await readFile(manifest), original)).toBe(0); expect(result.error.rollbackFailures).toEqual([]); expect(await readFile(firstBackup, "utf8")).toBe( "historical instructions V0\n", diff --git a/tests/boundary/filesystem/setupSkillPublication.test.ts b/tests/boundary/filesystem/setupSkillPublication.test.ts index 6923d3cda9..56d2d3bec3 100644 --- a/tests/boundary/filesystem/setupSkillPublication.test.ts +++ b/tests/boundary/filesystem/setupSkillPublication.test.ts @@ -87,14 +87,16 @@ describe("skill publication ownership", () => { expect.objectContaining({ path: manifest }), ]); const expected = change === "mode" ? published : userBytes; - expect(await readFile(manifest)).toEqual(expected); + if (expected === undefined) + throw new Error("Expected published skill bytes"); + expect(Buffer.compare(await readFile(manifest), expected)).toBe(0); if (existing) expect(await readFile(`${manifest}.rea.backup`, "utf8")).toBe( "original instructions\n", ); else expect(result.error.recoveryBackupPaths).toEqual([]); expect(await result.error.retryCleanup()).toEqual([]); - expect(await readFile(manifest)).toEqual(expected); + expect(Buffer.compare(await readFile(manifest), expected)).toBe(0); }, ); @@ -156,8 +158,10 @@ describe("skill publication recovery", () => { if (result.ok) throw new Error("Expected readback failure"); expect(result.error.stage).toBe("readback"); expect(result.error.reason).toBe("readback unavailable"); - expect(await readFile(manifest)).toEqual(bytes); - expect(await readFile(`${manifest}.rea.backup`)).toEqual(bytes); + expect(Buffer.compare(await readFile(manifest), bytes)).toBe(0); + expect( + Buffer.compare(await readFile(`${manifest}.rea.backup`), bytes), + ).toBe(0); if (process.platform !== "win32") expect((await stat(manifest)).mode & 0o777).toBe(0o640); expect( diff --git a/tests/boundary/filesystem/stableArtifact.test.ts b/tests/boundary/filesystem/stableArtifact.test.ts index d3616d111d..2b03a7a75a 100644 --- a/tests/boundary/filesystem/stableArtifact.test.ts +++ b/tests/boundary/filesystem/stableArtifact.test.ts @@ -24,9 +24,12 @@ it("reads exact binary and empty bytes and rejects symlink/size/cancellation bou const root = await createTestTempDirectory("rea-stable-artifact-"); const path = join(root, "source.bin"); await writeFile(path, Buffer.from([0, 255, 254])); - expect((await readStableArtifact(path, 3)).bytes).toEqual( - Buffer.from([0, 255, 254]), - ); + expect( + Buffer.compare( + (await readStableArtifact(path, 3)).bytes, + Buffer.from([0, 255, 254]), + ), + ).toBe(0); await expect(readStableArtifact(path, 2)).rejects.toMatchObject({ reason: "limit", }); diff --git a/tests/boundary/filesystem/staticCheckReports.test.ts b/tests/boundary/filesystem/staticCheckReports.test.ts index 10ef066744..c133e21cfd 100644 --- a/tests/boundary/filesystem/staticCheckReports.test.ts +++ b/tests/boundary/filesystem/staticCheckReports.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { access, cp, mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; @@ -151,6 +151,9 @@ it("excludes translation staging while retaining authored website errors", async const root = await fixture(); for (const name of [".gitignore", ".oxlintrc.json"]) await writeFile(join(root, name), await readFile(join(repository, name))); + await cp(join(repository, "scripts/lint"), join(root, "scripts/lint"), { + recursive: true, + }); const authored = "website/public/assets/authored.js"; const staged = "website/.i18n-regression/site/examples/notes-electron/main.js"; @@ -163,7 +166,7 @@ it("excludes translation staging while retaining authored website errors", async ); } - const result = invoke(root, "lint"); + const result = invoke(root, "lint", "website"); expect(result.error).toBeUndefined(); expect(result.status).toBe(1); expect(result.stdout).toContain("1 errors, 0 warnings"); diff --git a/tests/boundary/filesystem/targetDiagnostics.test.ts b/tests/boundary/filesystem/targetDiagnostics.test.ts index 5cf02b2452..4dd1f014aa 100644 --- a/tests/boundary/filesystem/targetDiagnostics.test.ts +++ b/tests/boundary/filesystem/targetDiagnostics.test.ts @@ -365,7 +365,7 @@ describe("caller-selected and inferred target interpretation", () => { ok: true, value: { kind: "database", format: "analysis-database" }, }); - expect(await readFile(databasePath)).toEqual(bytes); + expect(Buffer.compare(await readFile(databasePath), bytes)).toBe(0); }, ); diff --git a/tests/boundary/filesystem/verifierCommandOutput.test.ts b/tests/boundary/filesystem/verifierCommandOutput.test.ts index bc85ffcb7c..968085892b 100644 --- a/tests/boundary/filesystem/verifierCommandOutput.test.ts +++ b/tests/boundary/filesystem/verifierCommandOutput.test.ts @@ -37,8 +37,10 @@ it("writes exact Unicode and empty output bytes and returns the original output" expect(await saveVerifierCommandOutput(async () => output, paths)).toBe( output, ); - expect(await readFile(paths.stdout)).toEqual(Buffer.from(output.stdout)); - expect(await readFile(paths.stderr)).toEqual(Buffer.alloc(0)); + expect( + Buffer.compare(await readFile(paths.stdout), Buffer.from(output.stdout)), + ).toBe(0); + expect(Buffer.compare(await readFile(paths.stderr), Buffer.alloc(0))).toBe(0); }); it("retains both failed-command streams while preserving its exact cause", async () => { @@ -59,10 +61,13 @@ it("retains both failed-command streams while preserving its exact cause", async throw cause; }, paths), ).rejects.toBe(cause); - expect(await readFile(paths.stdout)).toEqual(Buffer.alloc(0)); - expect(await readFile(paths.stderr)).toEqual( - Buffer.from(cause.snapshot.stderr.text), - ); + expect(Buffer.compare(await readFile(paths.stdout), Buffer.alloc(0))).toBe(0); + expect( + Buffer.compare( + await readFile(paths.stderr), + Buffer.from(cause.snapshot.stderr.text), + ), + ).toBe(0); }); it("writes available partial failure output without replacing unavailable evidence", async () => { @@ -84,7 +89,7 @@ it("writes available partial failure output without replacing unavailable eviden throw cause; }, paths), ).rejects.toBe(cause); - expect(await readFile(paths.stdout)).toEqual(original); + expect(Buffer.compare(await readFile(paths.stdout), original)).toBe(0); expect(await readFile(paths.stderr, "utf8")).toBe(cause.snapshot.stderr.text); }); diff --git a/tests/boundary/filesystem/verifierFileSnapshots.test.ts b/tests/boundary/filesystem/verifierFileSnapshots.test.ts index 8b4a0736a2..65b8ed6ebb 100644 --- a/tests/boundary/filesystem/verifierFileSnapshots.test.ts +++ b/tests/boundary/filesystem/verifierFileSnapshots.test.ts @@ -44,7 +44,9 @@ it("preserves selected path spelling, UTF-8 bytes and missing-file facts", async { path: missing, exists: false, content: null }, { path, exists: true, content: Buffer.from(content, "utf8") }, ]); - expect(await readFile(path)).toEqual(Buffer.from(content, "utf8")); + expect( + Buffer.compare(await readFile(path), Buffer.from(content, "utf8")), + ).toBe(0); }); it("detects distinct invalid UTF-8 bytes in verifier file readback", async () => { diff --git a/tests/boundary/filesystem/webScriptExport.test.ts b/tests/boundary/filesystem/webScriptExport.test.ts index 74e08c6fce..24fd34ad11 100644 --- a/tests/boundary/filesystem/webScriptExport.test.ts +++ b/tests/boundary/filesystem/webScriptExport.test.ts @@ -105,13 +105,21 @@ describe("empty URL component publication", () => { for (const [index, script] of result.scripts.entries()) { if (script.content.state !== "exported") throw new Error("Missing source bytes"); - expect(script.url).toBe(scripts[index]?.url); + const expected = scripts[index]; + if (expected === undefined) throw new Error("Missing expected source"); + expect(script.url).toBe(expected.url); expect(script.content.layout).toBe(index === 0 ? "url-path" : "isolated"); expect( - await readFile( - join(result.analysis_input.input_path, script.content.relative_path), + Buffer.compare( + await readFile( + join( + result.analysis_input.input_path, + script.content.relative_path, + ), + ), + expected.bytes, ), - ).toEqual(scripts[index]?.bytes); + ).toBe(0); } const manifest = JSON.parse(await readFile(result.manifest.path, "utf8")); expect(manifest.scripts).toEqual(result.scripts); @@ -158,15 +166,16 @@ describe("captured script publication boundary", () => { const bytes = await readFile( join(result.analysis_input.input_path, script.content.relative_path), ); - expect(bytes).toEqual( - capture.events.items - .filter(({ kind }) => kind === "network-content") - .map((event) => - event.kind === "network-content" && event.body.state === "captured" - ? Buffer.from(event.body.content, "base64") - : Buffer.alloc(0), - )[index], - ); + const expected = capture.events.items + .filter(({ kind }) => kind === "network-content") + .map((event) => + event.kind === "network-content" && event.body.state === "captured" + ? Buffer.from(event.body.content, "base64") + : Buffer.alloc(0), + )[index]; + if (expected === undefined) + throw new Error("Missing expected script bytes"); + expect(bytes.equals(expected)).toBe(true); } const analyzed = await analyzeJavaScriptApplication(result.analysis_input); if (!analyzed.ok) throw analyzed.error; @@ -239,11 +248,10 @@ describe("captured script publication boundary", () => { expect(persisted.limitations).toContain( PLAYWRIGHT_RESPONSE_BODY_PROVENANCE_LIMITATION, ); - expect( - await readFile( - join(parsed.analysis_input.input_path, first.content.relative_path), - ), - ).toEqual(binary); + const exportedBytes = await readFile( + join(parsed.analysis_input.input_path, first.content.relative_path), + ); + expect(exportedBytes.equals(binary)).toBe(true); expect( await readFile( join(parsed.analysis_input.input_path, second.content.relative_path), @@ -1372,12 +1380,20 @@ it("isolates an observed exclusive-create alias using typed facts when diagnosti for (const [index, script] of result.scripts.entries()) { if (script.content.state !== "exported") throw new Error("Missing script bytes"); - expect(script.url).toBe(scripts[index]?.url); + const expected = scripts[index]; + if (expected === undefined) throw new Error("Missing expected script"); + expect(script.url).toBe(expected.url); expect( - await readFile( - join(result.analysis_input.input_path, script.content.relative_path), + Buffer.compare( + await readFile( + join( + result.analysis_input.input_path, + script.content.relative_path, + ), + ), + expected.bytes, ), - ).toEqual(scripts[index]?.bytes); + ).toBe(0); } } finally { write.mockRestore(); diff --git a/tests/boundary/firmware/firmwarePublicationPaths.test.ts b/tests/boundary/firmware/firmwarePublicationPaths.test.ts index 6ec6c7a700..6d21371da1 100644 --- a/tests/boundary/firmware/firmwarePublicationPaths.test.ts +++ b/tests/boundary/firmware/firmwarePublicationPaths.test.ts @@ -143,7 +143,7 @@ it("publishes the writer's normalized path while retaining source spelling and u ]); const published = result.files[0]; if (published === undefined) throw new Error("Expected published child"); - expect(await readFile(published.path)).toEqual(bytes); + expect(Buffer.compare(await readFile(published.path), bytes)).toBe(0); expect(result).toMatchObject({ coverage: "partial", extraction_gaps: [ diff --git a/tests/boundary/firmware/firmwareReportFiles.test.ts b/tests/boundary/firmware/firmwareReportFiles.test.ts index b1be22304d..d5c2e13357 100644 --- a/tests/boundary/firmware/firmwareReportFiles.test.ts +++ b/tests/boundary/firmware/firmwareReportFiles.test.ts @@ -277,7 +277,7 @@ it("does not activate an abort-bound source during awaited destination setup", a await expect(access(join(root, "published"))).rejects.toMatchObject({ code: "ENOENT", }); - expect(await readFile(source)).toEqual(bytes); + expect(Buffer.compare(await readFile(source), bytes)).toBe(0); }); it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( diff --git a/tests/boundary/firmware/firmwareWorkspaceRecovery.test.ts b/tests/boundary/firmware/firmwareWorkspaceRecovery.test.ts index 8193bf87be..dad2285010 100644 --- a/tests/boundary/firmware/firmwareWorkspaceRecovery.test.ts +++ b/tests/boundary/firmware/firmwareWorkspaceRecovery.test.ts @@ -113,7 +113,7 @@ it.skipIf(process.platform !== "linux" || process.getuid?.() === 0)( await fixture.close(); } await assertFirmwareCleanup(fixture.launches); - expect(await readFile(fixture.path)).toEqual(sourceBytes); + expect(Buffer.compare(await readFile(fixture.path), sourceBytes)).toBe(0); for (const output of [fixture.output, join(fixture.root, "second-output")]) expect(await readFile(join(output, "rootfs/config"), "utf8")).toBe( "firmware=true\n", diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index cf1f8c9071..a508d009ff 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -1021,7 +1021,9 @@ it("preserves standalone PTX BOM and trailing NUL bytes in the retained text art const bytes = Buffer.from("\uFEFF" + ptxFixture + "\0"); const result = await inspect(bytes); if (!result.ok) throw result.error; - expect(Buffer.from(result.value.ptx_artifacts[0]!.text)).toEqual(bytes); + expect( + Buffer.compare(Buffer.from(result.value.ptx_artifacts[0]!.text), bytes), + ).toBe(0); expect(result.value.ptx_artifacts[0]!.byte_length).toBe(bytes.length); }); it("rejects a changed full-file identity before applying positional object selection", async () => { diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index 224121527d..cf918e9812 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -232,7 +232,7 @@ it.each(["thin", "universal"] as const)( expect(linked.return_types).toEqual( link(createIl2cppBinaryFixture()).return_types, ); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); }, ); @@ -387,7 +387,7 @@ it("reads a PE pointer array across file bytes and the declared zero-fill tail", const bytes = createSplitPointerPeFixture(); const original = Buffer.from(bytes); expect(link(bytes).method_addresses).toEqual(expectedAddresses); - expect(bytes).toEqual(original); + expect(Buffer.compare(bytes, original)).toBe(0); expect(failure(createSplitPointerPeFixture(false)).reason).toBe( "code-gen-modules-not-found", ); diff --git a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts index 3688136b39..908b835a53 100644 --- a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts +++ b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts @@ -96,7 +96,7 @@ it("links one code-loaded type table among short runs across many mappings", () expect(result.types_field).toBeNull(); expect(result.types_count).toBe(4); expect(result.return_types.every((row) => row.name === "int")).toBe(true); - expect(bytes).toEqual(snapshot); + expect(Buffer.compare(bytes, snapshot)).toBe(0); }); it.each([183, 62])( @@ -110,7 +110,7 @@ it.each([183, 62])( ]); const snapshot = Buffer.from(split); expect(link(split)).toEqual(link(original)); - expect(split).toEqual(snapshot); + expect(Buffer.compare(split, snapshot)).toBe(0); }, ); @@ -136,7 +136,7 @@ it.each([false, true])( ), }), ); - expect(bytes).toEqual(snapshot); + expect(Buffer.compare(bytes, snapshot)).toBe(0); }, ); @@ -169,7 +169,7 @@ it("refuses an unresolved count relocation with conflicting file/zero-fill owner message: expect.stringContaining("ambiguous file and zero-fill mappings"), }), ); - expect(bytes).toEqual(snapshot); + expect(Buffer.compare(bytes, snapshot)).toBe(0); }); it("clears a file-backed unresolved count before scanning registration candidates", () => { @@ -184,7 +184,7 @@ it("clears a file-backed unresolved count before scanning registration candidate expect(() => link(bytes)).toThrow( expect.objectContaining({ reason: "code-registration-not-found" }), ); - expect(bytes).toEqual(snapshot); + expect(Buffer.compare(bytes, snapshot)).toBe(0); }); it.each([false, true])( diff --git a/tests/boundary/mcp/asarListingAdmissionMcp.test.ts b/tests/boundary/mcp/asarListingAdmissionMcp.test.ts index c8db05e921..19a5b07aab 100644 --- a/tests/boundary/mcp/asarListingAdmissionMcp.test.ts +++ b/tests/boundary/mcp/asarListingAdmissionMcp.test.ts @@ -39,7 +39,7 @@ it("reports expanded ASAR admission failure through MCP and permits a valid retr }, }); expect(result.structuredContent).toBeUndefined(); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); const valid = join(root, "valid.asar"); await writeFile(valid, nestedAsarHeader(1, "lib", 1)); expect( diff --git a/tests/boundary/mcp/asarUnpackedExtractionMcp.test.ts b/tests/boundary/mcp/asarUnpackedExtractionMcp.test.ts index fdb8436cee..a671bc783f 100644 --- a/tests/boundary/mcp/asarUnpackedExtractionMcp.test.ts +++ b/tests/boundary/mcp/asarUnpackedExtractionMcp.test.ts @@ -68,7 +68,9 @@ it("extracts a rewritten unpacked companion only when the caller records the mis onTestFinished(() => rm(result.output_root, { recursive: true, force: true }), ); - expect(await readFile(join(result.output_root, addon))).toEqual(stripped); + expect( + Buffer.compare(await readFile(join(result.output_root, addon)), stripped), + ).toBe(0); expect(result.integrity_contradictions).toEqual([ { logical_path: addon, trust: "observed-untrusted" }, ]); diff --git a/tests/boundary/mcp/elfNotesMcp.test.ts b/tests/boundary/mcp/elfNotesMcp.test.ts index 6590dfdc00..63686bc91d 100644 --- a/tests/boundary/mcp/elfNotesMcp.test.ts +++ b/tests/boundary/mcp/elfNotesMcp.test.ts @@ -103,7 +103,7 @@ mcpTest( examined_tables: 1, examined_entries: 1, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } finally { await session.close(); } @@ -130,7 +130,7 @@ mcpTest( "Compressed ELF note sections are unsupported", ), }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); } finally { await session.close(); } diff --git a/tests/boundary/mcp/environmentBundleMcp.test.ts b/tests/boundary/mcp/environmentBundleMcp.test.ts index 5f8211e608..fd07fab79f 100644 --- a/tests/boundary/mcp/environmentBundleMcp.test.ts +++ b/tests/boundary/mcp/environmentBundleMcp.test.ts @@ -118,9 +118,13 @@ it("writes the bundle environment-build writes from retained capture Evidence", const files = await listFiles(input.output_directory); expect(files).toEqual(await listFiles(join(root, "reference"))); for (const path of files) - expect(await readFile(join(input.output_directory, path)), path).toEqual( - await readFile(join(root, "reference", path)), - ); + expect( + Buffer.compare( + await readFile(join(input.output_directory, path)), + await readFile(join(root, "reference", path)), + ), + path, + ).toBe(0); expect(report.coverage).toEqual(reference.value.coverage); }); @@ -154,8 +158,11 @@ it("copies a capture Evidence file byte for byte and plans without writing on a }); expect(built.isError).not.toBe(true); expect( - await readFile(join(root, "built/envs/grafana-example/evidence.json")), - ).toEqual(await readFile(path)); + Buffer.compare( + await readFile(join(root, "built/envs/grafana-example/evidence.json")), + await readFile(path), + ), + ).toBe(0); }); it("refuses an existing or parentless output directory and leaves it as it was", async () => { diff --git a/tests/boundary/mcp/ghidraReopen.test.ts b/tests/boundary/mcp/ghidraReopen.test.ts index f7c584ae0e..ac2dbcce7c 100644 --- a/tests/boundary/mcp/ghidraReopen.test.ts +++ b/tests/boundary/mcp/ghidraReopen.test.ts @@ -186,7 +186,7 @@ const connectGhidra = async () => { } } for (const launched of launcher.launches) await expectReleased(launched); - expect(await readFile(target)).toEqual(source); + expect(Buffer.compare(await readFile(target), source)).toBe(0); }; try { await server.connect(serverTransport); diff --git a/tests/boundary/mcp/goBinaryViews.test.ts b/tests/boundary/mcp/goBinaryViews.test.ts index a319b737bc..0788c62dd6 100644 --- a/tests/boundary/mcp/goBinaryViews.test.ts +++ b/tests/boundary/mcp/goBinaryViews.test.ts @@ -183,7 +183,7 @@ goViewTest( }, }, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); await unlink(path); const source = { kind: "retained-evidence", diff --git a/tests/boundary/mcp/peLoadConfigMcp.test.ts b/tests/boundary/mcp/peLoadConfigMcp.test.ts index 8a347e9bd5..f749fbeebe 100644 --- a/tests/boundary/mcp/peLoadConfigMcp.test.ts +++ b/tests/boundary/mcp/peLoadConfigMcp.test.ts @@ -57,7 +57,7 @@ mcpTest( dynamic_value_reloc_table_section: PE_LOAD_CONFIG_FIXTURE.dynamicRelocSection, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); const bad = await client.callTool( { name: "inspect_pe_load_config", diff --git a/tests/boundary/mcp/selectedSnapshotCleanupMcp.test.ts b/tests/boundary/mcp/selectedSnapshotCleanupMcp.test.ts index 7b654f7f76..54bdf34cbb 100644 --- a/tests/boundary/mcp/selectedSnapshotCleanupMcp.test.ts +++ b/tests/boundary/mcp/selectedSnapshotCleanupMcp.test.ts @@ -175,8 +175,8 @@ it("retains a selected snapshot descriptor through MCP cleanup discovery and sam expect(replayEvidence.evidence_id).not.toBe(originalEvidence.evidence_id); expect(starts).toBe(priorStarts); expect(calls).toEqual(priorCalls); - expect(await readFile(snapshotPath)).toEqual(snapshotBytes); - expect(await readFile(first)).toEqual(targetBytes); + expect(Buffer.compare(await readFile(snapshotPath), snapshotBytes)).toBe(0); + expect(Buffer.compare(await readFile(first), targetBytes)).toBe(0); } finally { restoreClose(); try { diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index 0cfb8bebed..2371e9186f 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -134,7 +134,7 @@ describe("MCP snapshot lifecycle ordering", () => { }) ).isError, ).toBe(true); - expect(await readFile(snapshotPath)).toEqual(bytes); + expect(Buffer.compare(await readFile(snapshotPath), bytes)).toBe(0); expect( ( await mcp.callTool({ diff --git a/tests/boundary/mcp/wasmArtifactMcp.test.ts b/tests/boundary/mcp/wasmArtifactMcp.test.ts index 238dbddf1a..9413177b72 100644 --- a/tests/boundary/mcp/wasmArtifactMcp.test.ts +++ b/tests/boundary/mcp/wasmArtifactMcp.test.ts @@ -161,9 +161,12 @@ it.each(["bin", "hop"])( }); await client.ping(); expect(createClient).not.toHaveBeenCalled(); - expect(await readFile(path)).toEqual( - Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0]), - ); + expect( + Buffer.compare( + await readFile(path), + Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0]), + ), + ).toBe(0); }, ); diff --git a/tests/boundary/mcp/webScriptSelectionMcp.test.ts b/tests/boundary/mcp/webScriptSelectionMcp.test.ts index f2ac5aa468..6bfbe233b0 100644 --- a/tests/boundary/mcp/webScriptSelectionMcp.test.ts +++ b/tests/boundary/mcp/webScriptSelectionMcp.test.ts @@ -189,10 +189,13 @@ it("publishes script bytes and retains the returned Evidence without starting an if (exported.analysis_input === null) throw new Error("published script bytes have no analysis input root"); expect( - await readFile( - join(exported.analysis_input.input_path, content.relative_path), + Buffer.compare( + await readFile( + join(exported.analysis_input.input_path, content.relative_path), + ), + bytes, ), - ).toEqual(bytes); + ).toBe(0); expect( JSON.parse(await readFile(exported.manifest.path, "utf8")), ).toMatchObject({ diff --git a/tests/boundary/pcap/pcapProvider.test.ts b/tests/boundary/pcap/pcapProvider.test.ts index e0cac88901..ca4e27afcb 100644 --- a/tests/boundary/pcap/pcapProvider.test.ts +++ b/tests/boundary/pcap/pcapProvider.test.ts @@ -310,7 +310,9 @@ it.each([ }, }); expect(producerLaunches).toBe(1); - expect(await readFile(path)).toEqual(Buffer.from("pcap-fixture-bytes")); + expect( + Buffer.compare(await readFile(path), Buffer.from("pcap-fixture-bytes")), + ).toBe(0); const snapshotPath = result.error.cleanupResources.find((resource) => resource.endsWith("/capture.pcap"), ); @@ -340,9 +342,12 @@ it.each([ }); expect(result.error.cleanupIncomplete).toBe(true); expect(cleanupAttempts).toBe(attemptsBeforeClose + 1); - await expect(readFile(snapshotPath)).resolves.toEqual( - Buffer.from("pcap-fixture-bytes"), - ); + expect( + Buffer.compare( + await readFile(snapshotPath), + Buffer.from("pcap-fixture-bytes"), + ), + ).toBe(0); } finally { denied.mockRestore(); } @@ -514,7 +519,7 @@ console.log(${JSON.stringify(TSHARK_JSON)});`, expect(projectAnalysisError(result.error)).toMatchObject({ code: "artifact_changed", }); - expect(await readFile(path)).toEqual(before); + expect(Buffer.compare(await readFile(path), before)).toBe(0); }); it("binds both tools to the snapshot when the original capture changes", async () => { diff --git a/tests/boundary/process/modelFixtureShutdown.test.ts b/tests/boundary/process/modelFixtureShutdown.test.ts index 28ab529e6d..2673022ed3 100644 --- a/tests/boundary/process/modelFixtureShutdown.test.ts +++ b/tests/boundary/process/modelFixtureShutdown.test.ts @@ -244,7 +244,7 @@ for (const protocol of protocols) expect(closed.cause).toBe(lateFailure); } else expect(closed).toEqual({ kind: "fixture-closed" }); expect(callbackFinished).toBe(true); - expect(await readFile(artifactPath)).toEqual(artifact); + expect(Buffer.compare(await readFile(artifactPath), artifact)).toBe(0); expect(fixture.failure).toBe( outcome === "late-error" ? String(lateFailure) : undefined, ); @@ -394,7 +394,7 @@ for (const protocol of protocols) await bounded(Promise.all(closing), "both admitted fixture closes"), ).toEqual([{ kind: "fixture-closed" }, { kind: "fixture-closed" }]); expect(callbackFinished).toBe(true); - expect(await readFile(artifactPath)).toEqual(artifact); + expect(Buffer.compare(await readFile(artifactPath), artifact)).toBe(0); expect(fixture.failure).toBe(ordinaryFailure); expect(server.listening).toBe(false); expect(server.address()).toBeNull(); diff --git a/tests/boundary/process/nativeVerifierCommandOwnership.test.ts b/tests/boundary/process/nativeVerifierCommandOwnership.test.ts index a11bce0f0a..fd1ffdc60b 100644 --- a/tests/boundary/process/nativeVerifierCommandOwnership.test.ts +++ b/tests/boundary/process/nativeVerifierCommandOwnership.test.ts @@ -211,7 +211,7 @@ it.skipIf(process.platform === "win32")( expect(report.failure.stderr).toBe("native verifier diagnostics\n"); expect(report.labExists).toBe(true); expect(report.artifactAtCompletion).toBe(artifact.toString("base64")); - expect(await readFile(artifactPath)).toEqual(artifact); + expect(Buffer.compare(await readFile(artifactPath), artifact)).toBe(0); expect(report.liveAtCompletion).toEqual([]); } finally { await writeFile( diff --git a/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts b/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts index 2aaee785df..22785b59ce 100644 --- a/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts +++ b/tests/boundary/process/nativeVerifierPreflightOwnership.test.ts @@ -242,7 +242,9 @@ it.skipIf(process.platform === "win32").each(clients)( expect(receipt.artifacts).toBe(ready.lab); if (receipt.requests !== undefined) expect(receipt.requests).toEqual([]); // Read only after the verifier's own cleanup, before removing test scratch. - expect(await readFile(ready.artifact)).toEqual(retainedBytes); + expect( + Buffer.compare(await readFile(ready.artifact), retainedBytes), + ).toBe(0); } finally { // Self-release the authored orphan even on RED. PID observations never // authorize signals; outer cleanup retains its canonical acquisition proof. diff --git a/tests/boundary/process/packagePackIsolation.test.ts b/tests/boundary/process/packagePackIsolation.test.ts index 8d6875ae8a..9c3d95aa3c 100644 --- a/tests/boundary/process/packagePackIsolation.test.ts +++ b/tests/boundary/process/packagePackIsolation.test.ts @@ -83,6 +83,6 @@ process.stdout.write(JSON.stringify({ failure })); expect(JSON.parse(stdout)).toEqual({ failure: expectedFailure }); expect((await readdir(packageRoot)).sort()).toEqual(packageContentsBefore); - expect(await readFile(archivePath)).toEqual(sentinel); + expect(Buffer.compare(await readFile(archivePath), sentinel)).toBe(0); await expect(access(workspace)).rejects.toMatchObject({ code: "ENOENT" }); }); diff --git a/tests/boundary/process/packedBridgeVerification.test.ts b/tests/boundary/process/packedBridgeVerification.test.ts index 3dba30d08f..dc5aceb5bb 100644 --- a/tests/boundary/process/packedBridgeVerification.test.ts +++ b/tests/boundary/process/packedBridgeVerification.test.ts @@ -221,7 +221,9 @@ it.each(["valid", "invalid"] as const)( expectedPythonCommandFailure("SyntaxError"), ); expect(await readdir(root.path)).toEqual(["pack"]); - expect(await readFile(tarball)).toEqual(await readFile(input.tarball)); + expect( + Buffer.compare(await readFile(tarball), await readFile(input.tarball)), + ).toBe(0); }, [() => (commands === undefined ? root.close() : commands.close())]); await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); }, diff --git a/tests/boundary/process/verifierCommandSemantics.test.ts b/tests/boundary/process/verifierCommandSemantics.test.ts index 683eb805e4..3624b93eb3 100644 --- a/tests/boundary/process/verifierCommandSemantics.test.ts +++ b/tests/boundary/process/verifierCommandSemantics.test.ts @@ -133,7 +133,7 @@ process.stdin.resume(); expect(output?.stderr).toBe("closed input diagnostics\n"); expect(commands.cleanupResources).toEqual([]); expect(existsSync(directory)).toBe(true); - expect(await readFile(artifact)).toEqual(bytes); + expect(Buffer.compare(await readFile(artifact), bytes)).toBe(0); }, ); @@ -183,7 +183,7 @@ it("reports a numeric nonzero exit with its complete output snapshot after clean }); expect(commands.cleanupResources).toEqual([]); expect(existsSync(directory)).toBe(true); - expect(await readFile(artifact)).toEqual(bytes); + expect(Buffer.compare(await readFile(artifact), bytes)).toBe(0); }); it("retries only the failed exact file capability after a peer releases, retaining both evidence files", async () => { @@ -259,8 +259,8 @@ it("retries only the failed exact file capability after a peer releases, retaini expect(peerHandle.fd).toBe(-1); expect(commands.cleanupResources).toEqual([]); expect(existsSync(directory)).toBe(true); - expect(await readFile(artifact)).toEqual(bytes); - expect(await readFile(peerArtifact)).toEqual(bytes); + expect(Buffer.compare(await readFile(artifact), bytes)).toBe(0); + expect(Buffer.compare(await readFile(peerArtifact), bytes)).toBe(0); }); it("refuses new owners and commands after successful close while leaving new resources with their caller", async () => { @@ -502,7 +502,7 @@ it.skipIf(process.platform === "win32")( ), ).toBe(false); expect(existsSync(directory)).toBe(true); - expect(await readFile(artifact)).toEqual(bytes); + expect(Buffer.compare(await readFile(artifact), bytes)).toBe(0); } finally { deny = false; release.resolve(); diff --git a/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts b/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts index c1547b38b0..b697d70f00 100644 --- a/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts +++ b/tests/boundary/providers/adb/adbHashCleanupConsumers.test.ts @@ -149,9 +149,12 @@ test.each(digestCases)( await f.provider.close(); expect(returnedError.cleanupIncomplete).toBe(false); expect(returnedError.cause).toBe(originalCause); - await expect(readFile(retainedPath(f.root))).resolves.toEqual( - Buffer.from("pulled bytes"), - ); + expect( + Buffer.compare( + await readFile(retainedPath(f.root)), + Buffer.from("pulled bytes"), + ), + ).toBe(0); }, ); diff --git a/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts b/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts index a084f5a187..e99702ff39 100644 --- a/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts +++ b/tests/boundary/providers/adb/adbInputSnapshotOwnership.test.ts @@ -144,9 +144,9 @@ test("retains failed snapshot cleanup ownership and retries it on close", async sha256: f.digest, }, }); - await expect(readFile(f.snapshotPath())).resolves.toEqual( - await readFile(f.source), - ); + expect( + Buffer.compare(await readFile(f.snapshotPath()), await readFile(f.source)), + ).toBe(0); await f.provider.close(); await expect(lstat(dirname(f.snapshotPath()))).rejects.toMatchObject({ @@ -372,9 +372,10 @@ test.each([ const secondary = result.error.secondaryFailures[0]; expect(secondary).toBeInstanceOf(AnalysisOutputError); expect(secondary?.cause).toBeInstanceOf(AdbInputSnapshotFailure); - expect( - (secondary?.cause as AdbInputSnapshotFailure).cleanupCause, - ).toBeDefined(); + const snapshotFailure = secondary?.cause; + if (!(snapshotFailure instanceof AdbInputSnapshotFailure)) + throw new Error("Expected snapshot cleanup failure"); + expect(snapshotFailure.cleanupCause).toBeDefined(); }, ); @@ -464,7 +465,7 @@ test.each([ }), }, }); - await expect(readFile(localPath)).resolves.toEqual(pulledBytes); + expect(Buffer.compare(await readFile(localPath), pulledBytes)).toBe(0); const originatingError = result.error; await f.provider.close(); expect(closeAttempts).toBe(failClose ? 2 : 1); @@ -681,7 +682,12 @@ test.each([false, true])( dirname(f.snapshotPath()), ); else expect(result.error.cleanupIncomplete).toBe(false); - expect(await readFile(f.source)).toEqual(Buffer.from("immutable payload")); + expect( + Buffer.compare( + await readFile(f.source), + Buffer.from("immutable payload"), + ), + ).toBe(0); await f.provider.close(); }, ); diff --git a/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts b/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts index 492dde7be2..f8e018cb6e 100644 --- a/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts +++ b/tests/boundary/providers/adb/adbScreenPublicationCancellation.test.ts @@ -124,7 +124,9 @@ test("retains a committed screen capture receipt when cancellation arrives after height: 1, }, }); - expect(await readFile(join(output, "screen.png"))).toEqual(f.png); + expect( + Buffer.compare(await readFile(join(output, "screen.png")), f.png), + ).toBe(0); await f.provider.close(); }); @@ -279,7 +281,7 @@ test("retains the committed receipt and cleanup observation when cancellation ra confidence: "observed", normalized_result: { local_path: destination, bytes: f.png.byteLength }, }); - expect(await readFile(destination)).toEqual(f.png); + expect(Buffer.compare(await readFile(destination), f.png)).toBe(0); await provider.close(); expect(closeAttempts).toBe(2); expect(result.error.cleanupIncomplete).toBe(false); @@ -300,7 +302,9 @@ test("publishes screen captures when the output filesystem does not support hard input: { serial: "device", output_directory: output }, }); expect(result.ok).toBe(true); - expect(await readFile(join(output, "screen.png"))).toEqual(f.png); + expect( + Buffer.compare(await readFile(join(output, "screen.png")), f.png), + ).toBe(0); await f.provider.close(); }); @@ -401,7 +405,7 @@ test("retains a receipt when cancellation arrives after exclusive output creatio confidence: "observed", normalized_result: { local_path: destination, bytes: f.png.byteLength }, }); - expect(await readFile(destination)).toEqual(f.png); + expect(Buffer.compare(await readFile(destination), f.png)).toBe(0); await f.provider.close(); }); @@ -458,7 +462,7 @@ test("closes and retains ownership when exclusive output stat fails", async () = expect(result.error.cleanupResources).toContain(destination); expect(closeAttempts).toBe(0); expect(handleOpen).toBe(true); - expect(await readFile(destination)).toEqual(Buffer.alloc(0)); + expect(Buffer.compare(await readFile(destination), Buffer.alloc(0))).toBe(0); vi.mocked(rm).mockImplementation(async (path, options) => { if (String(path) === destination && ++removeAttempts === 1) throw new Error("controlled rollback removal fault"); @@ -598,7 +602,7 @@ test.each([false, true])( }); expect(result.error.cleanupIncomplete).toBe(rootFails); if (rootFails) expect(result.error.cleanupResources).toEqual([runtimePath]); - expect(await readFile(f.destination)).toEqual(f.png); + expect(Buffer.compare(await readFile(f.destination), f.png)).toBe(0); expect(f.closeAttempts()).toBe(2); await f.provider.close(); expect(result.error.cleanupIncomplete).toBe(false); @@ -627,7 +631,7 @@ test("keeps a sticky publication close failure until provider close retries its confidence: "observed", normalized_result: { local_path: f.destination, bytes: f.png.byteLength }, }); - expect(await readFile(f.destination)).toEqual(f.png); + expect(Buffer.compare(await readFile(f.destination), f.png)).toBe(0); expect(f.closeAttempts()).toBe(2); const linkedFailure = (result.error.cause as AggregateError).errors[0] as { readonly cleanupIncomplete: boolean; @@ -638,7 +642,7 @@ test("keeps a sticky publication close failure until provider close retries its expect(f.closeAttempts()).toBe(3); expect(linkedFailure.cleanupIncomplete).toBe(false); expect(result.error.cleanupIncomplete).toBe(false); - expect(await readFile(f.destination)).toEqual(f.png); + expect(Buffer.compare(await readFile(f.destination), f.png)).toBe(0); }); test("resolves an admission cleanup error when the retained runtime-root lease closes", async () => { @@ -668,7 +672,7 @@ test("resolves an admission cleanup error when the retained runtime-root lease c expect(f.closeAttempts()).toBe(4); expect(admissionError.cleanupIncomplete).toBe(false); expect(admissionError.cause).toBeInstanceOf(AggregateError); - expect(await readFile(f.destination)).toEqual(f.png); + expect(Buffer.compare(await readFile(f.destination), f.png)).toBe(0); }); test("resolves the provider close aggregate after its failed root owner recovers", async () => { @@ -691,7 +695,7 @@ test("resolves the provider close aggregate after its failed root owner recovers expect(f.closeAttempts()).toBe(4); expect(closeError).toMatchObject({ cleanupIncomplete: false }); expect(result.error.cleanupIncomplete).toBe(false); - expect(await readFile(f.destination)).toEqual(f.png); + expect(Buffer.compare(await readFile(f.destination), f.png)).toBe(0); }); test("retains incomplete cleanup on a cancelled capture while keeping cancellation typed", async () => { diff --git a/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts b/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts index db7d3ec8f9..19e2574bae 100644 --- a/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts +++ b/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts @@ -19,11 +19,11 @@ describe("Ghidra target snapshot", () => { platform: "linux", }); expect(snapshot.admission).toBeUndefined(); - await expect(readFile(snapshot.path)).resolves.toEqual(bytes); + expect(Buffer.compare(await readFile(snapshot.path), bytes)).toBe(0); await expect( createGhidraTargetSnapshot(source, root, sha256, { platform: "linux" }), ).rejects.toMatchObject({ code: "EEXIST" }); - await expect(readFile(snapshot.path)).resolves.toEqual(bytes); + expect(Buffer.compare(await readFile(snapshot.path), bytes)).toBe(0); await expect( createGhidraTargetSnapshot(source, root, sha256, { platform: "win32" }), ).rejects.toThrow(/No native private runtime owns/u); diff --git a/tests/boundary/providers/hopper/bridgeLauncher.test.ts b/tests/boundary/providers/hopper/bridgeLauncher.test.ts index e72c7348dc..ac72a5b3a2 100644 --- a/tests/boundary/providers/hopper/bridgeLauncher.test.ts +++ b/tests/boundary/providers/hopper/bridgeLauncher.test.ts @@ -878,7 +878,7 @@ it("ordinary executable launches pass a verified private image with configured l expect(supervisor.snapshot().stdout.text).toBe(preparedPath); expect(launched.value.preparedImagePath).toBe(preparedPath); await writeFile(targetPath, "replacement bytes"); - expect(await readFile(preparedPath)).toEqual(selected); + expect(Buffer.compare(await readFile(preparedPath), selected)).toBe(0); } finally { await supervisor.stop(); await launched.value.releaseLease?.(); diff --git a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts index 89696d48e0..aa0371d389 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts @@ -328,9 +328,12 @@ it("preserves every equal-content occurrence and copies a nested ASAR as its con extracted.artifacts[2]?.artifact_id, ); for (const path of ["a.txt", "b.txt", "app.asar"]) - expect(await readFile(join(output, path))).toEqual( - await readFile(join(source, path)), - ); + expect( + Buffer.compare( + await readFile(join(output, path)), + await readFile(join(source, path)), + ), + ).toBe(0); }); it("ignores unavailable members of a nested ASAR but refuses them as active entries", async () => { @@ -358,9 +361,12 @@ it("ignores unavailable members of a nested ASAR but refuses them as active entr expect( nestedExtraction.artifacts.map(({ relative_path }) => relative_path), ).toEqual(["app.asar"]); - expect(await readFile(join(nestedOutput, "app.asar"))).toEqual( - await readFile(archive), - ); + expect( + Buffer.compare( + await readFile(join(nestedOutput, "app.asar")), + await readFile(archive), + ), + ).toBe(0); const activeOutput = join(root, "active-output"); const activeResult = await extract(archive, activeOutput); diff --git a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts index eedf1eb80f..4e210a3040 100644 --- a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts +++ b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts @@ -117,7 +117,7 @@ describe("artifact Mach-O slices", () => { const chunks: Buffer[] = []; const stream = await reader.open({ ...secondEntry }); for await (const chunk of stream) chunks.push(Buffer.from(chunk)); - expect(Buffer.concat(chunks)).toEqual(Buffer.from(arm)); + expect(Buffer.compare(Buffer.concat(chunks), Buffer.from(arm))).toBe(0); expect(reader.provenance()).toEqual([ expect.objectContaining({ tool: "lipo", effects: ["read"] }), ]); diff --git a/tests/boundary/providers/native/artifactProvider.zipCancellation.test.ts b/tests/boundary/providers/native/artifactProvider.zipCancellation.test.ts index 8e8a4d781a..8af0ae0ccb 100644 --- a/tests/boundary/providers/native/artifactProvider.zipCancellation.test.ts +++ b/tests/boundary/providers/native/artifactProvider.zipCancellation.test.ts @@ -102,9 +102,12 @@ it("preserves a real ZIP stream's caller cancellation through extraction rollbac containment_verified: true, artifacts: [expect.objectContaining({ relative_path: "member.bin" })], }); - expect(await readFile(join(controlOutput, "member.bin"))).toEqual( - Buffer.from(bytes), - ); + expect( + Buffer.compare( + await readFile(join(controlOutput, "member.bin")), + Buffer.from(bytes), + ), + ).toBe(0); expect(await client.close()).toMatchObject({ ok: true }); } finally { writeSpy.mockRestore(); diff --git a/tests/boundary/providers/native/nativePlistTextSource.test.ts b/tests/boundary/providers/native/nativePlistTextSource.test.ts index b16898cd5c..b077cc8f2a 100644 --- a/tests/boundary/providers/native/nativePlistTextSource.test.ts +++ b/tests/boundary/providers/native/nativePlistTextSource.test.ts @@ -212,5 +212,5 @@ it.for([ (item) => item.format === "json" && item.exitCode === 1, ), ); - expect(await readFile(path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(path), fixture.bytes)).toBe(0); }); diff --git a/tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts b/tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts index 8c05be76a5..83368d9479 100644 --- a/tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts +++ b/tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts @@ -414,7 +414,9 @@ it("does not classify an unrelated path-echo diagnostic as an unsigned observati snapshotPath = args.at(-1); if (snapshotPath === undefined) throw new Error("Expected the bound snapshot as the codesign operand"); - expect(await readFile(snapshotPath)).toEqual(Buffer.from(CONTENT)); + expect( + Buffer.compare(await readFile(snapshotPath), Buffer.from(CONTENT)), + ).toBe(0); return new NativeFixtureRunner( { codesign: `${snapshotPath}: No such file or directory\n` }, 1, @@ -475,7 +477,7 @@ it.skipIf(process.platform !== "darwin")( ok: false, error: { _tag: "AnalysisArtifactChangedError" }, }); - expect(await readFile(path)).toEqual(originalBytes); + expect(Buffer.compare(await readFile(path), originalBytes)).toBe(0); if (snapshotPath === undefined) throw new Error("Expected codesign capture"); await expect(lstat(dirname(snapshotPath))).rejects.toMatchObject({ @@ -569,7 +571,9 @@ it("copies and hashes the same bytes across multiple snapshot chunks", async () target, new ArtifactResourceScope(), async (binding) => { - expect(await readFile(binding.snapshotPath)).toEqual(bytes); + expect(Buffer.compare(await readFile(binding.snapshotPath), bytes)).toBe( + 0, + ); const verified = await verifySignatureTarget(target, binding); return verified.ok ? ok(signatureInspectionFixture()) : verified; }, @@ -658,9 +662,12 @@ it.skipIf(process.platform !== "darwin")( await rename(selected, saved); await rename(replacement, selected); try { - expect(await readFile(path)).not.toEqual( - await readFile(join(saved, "program")), - ); + expect( + Buffer.compare( + await readFile(path), + await readFile(join(saved, "program")), + ), + ).not.toBe(0); return await real.run(tool, args, options); } finally { await rename(selected, replacement); diff --git a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts index d29cdfd69e..1f95aad8b0 100644 --- a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts +++ b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts @@ -525,7 +525,9 @@ linux( ok: false, error: { reason: "verification_unavailable" }, }); - expect(await readFile(join(f.cache, "release.json"))).toEqual(metadata); + expect( + Buffer.compare(await readFile(join(f.cache, "release.json")), metadata), + ).toBe(0); expect( (await readdir(f.publisher.cacheRoot)).some((name) => name.includes(".backup-"), diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 4e6f26fed2..8757aefb8b 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -351,7 +351,7 @@ it.each([ expect(report.annotations_text).toBe(`${source.toString("hex")}\n`); expect(report.tool.arguments).toContain(`--input-file=./${recordingName}`); expect(report.tool.arguments).not.toContain("--input-file=-"); - expect(await readFile(f.path)).toEqual(source); + expect(Buffer.compare(await readFile(f.path), source)).toBe(0); expect((await readdir(f.directory)).sort()).toEqual( [recordingName, "producer.mjs"].sort(), ); diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index 8044a9ada4..44ce4ca8fa 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -769,6 +769,6 @@ it.each(["inventory", "selected_table", "database"] as const)( ), }), ); - expect(await readFile(path)).toEqual(before); + expect(Buffer.compare(await readFile(path), before)).toBe(0); }, ); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index b39a3d2462..cd561c3c02 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -102,7 +102,7 @@ it("enforces a selected worker budget and completes the corrected same-database schema: { completeness: "complete" }, }); await expectPrivateRootRemoved(root); - expect(await readFile(database.path)).toEqual(original); + expect(Buffer.compare(await readFile(database.path), original)).toBe(0); }); nativeTest( @@ -179,7 +179,7 @@ nativeTest( ).toHaveLength(2); expect(result.error.cleanupResources).toEqual([]); await expectPrivateRootRemoved(root); - expect(await readFile(database.path)).toEqual(original); + expect(Buffer.compare(await readFile(database.path), original)).toBe(0); }, ); @@ -504,7 +504,7 @@ it.each(["late-cancellation", "cleanup-failure", "cancelled cleanup-failure"])( await provider.close(); await expectPrivateRootRemoved(root.path); } - expect(await readFile(database.path)).toEqual(original); + expect(Buffer.compare(await readFile(database.path), original)).toBe(0); }, ); @@ -586,7 +586,9 @@ it("inspects committed WAL schema and rows while preserving every source artifac }, }); for (const artifact of original) { - expect(await readFile(artifact.path)).toEqual(artifact.bytes); + expect(Buffer.compare(await readFile(artifact.path), artifact.bytes)).toBe( + 0, + ); const after = await lstat(artifact.path); expect({ ino: after.ino, @@ -901,7 +903,7 @@ it("attributes inventory selections beyond metadata bind capacity to result-colu }, }); expect(JSON.stringify(projected)).not.toContain("invalid_format"); - expect(await readFile(database.path)).toEqual(original); + expect(Buffer.compare(await readFile(database.path), original)).toBe(0); }); it("reports application-collation row failures as unavailable while retaining schema-only recovery", async () => { @@ -945,7 +947,7 @@ it("reports application-collation row failures as unavailable while retaining sc "Select columns whose values were read", ); } - expect(await readFile(path)).toEqual(original); + expect(Buffer.compare(await readFile(path), original)).toBe(0); }); it("rejects malformed databases without altering the selected file", async () => { @@ -961,7 +963,7 @@ it("rejects malformed databases without altering the selected file", async () => ok: false, error: { _tag: "AnalysisInputError" }, }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); nativeTest.runIf(process.platform !== "win32")( @@ -997,8 +999,8 @@ nativeTest( ok: false, error: { _tag: "AnalysisInputError" }, }); - expect(await readFile(database.path)).toEqual(main); - expect(await readFile(journalPath)).toEqual(journal); + expect(Buffer.compare(await readFile(database.path), main)).toBe(0); + expect(Buffer.compare(await readFile(journalPath), journal)).toBe(0); } finally { writer.exec("ROLLBACK"); writer.close(); @@ -1217,7 +1219,7 @@ it("retains an unavailable virtual module definition without executing it", asyn expect( await provider.inspect({ path, table: "optional_search" }), ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); - expect(await readFile(path)).toEqual(bytes); + expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); nativeTest( diff --git a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts index 3687ffc72a..cbefb2b2c5 100644 --- a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts @@ -67,7 +67,7 @@ it.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE).each(["PERSIST", "TRUNCATE"])( table: "records", }); expect(result.rows).toMatchObject({ returned_rows: 4, truncated: false }); - expect(await readFile(journalPath)).toEqual(before); + expect(Buffer.compare(await readFile(journalPath), before)).toBe(0); const afterStat = await lstat(journalPath, { bigint: true }); expect([ afterStat.ino, @@ -181,8 +181,8 @@ it("refuses an active journal even before SQLite writes its magic header", async reason: "format", message: expect.stringContaining("recovery"), }); - expect(await readFile(database.path)).toEqual(original); - expect(await readFile(journalPath)).toEqual(journal); + expect(Buffer.compare(await readFile(database.path), original)).toBe(0); + expect(Buffer.compare(await readFile(journalPath), journal)).toBe(0); } finally { writer.exec("ROLLBACK"); writer.close(); @@ -279,7 +279,9 @@ it("rejects a database changed after its copy but before WAL acquisition", async reason: "integrity", }); expect(changed).toBe(true); - expect(await readFile(`${root}/database.snapshot`)).toEqual(initial); + expect( + Buffer.compare(await readFile(`${root}/database.snapshot`), initial), + ).toBe(0); }); it.each(["-wal", "-journal"] as const)( diff --git a/tests/fixtures/machoArmSlices.ts b/tests/fixtures/machoArmSlices.ts index d22631faf6..a90d06c935 100644 --- a/tests/fixtures/machoArmSlices.ts +++ b/tests/fixtures/machoArmSlices.ts @@ -97,7 +97,7 @@ export const expectArmSliceExtraction = async ( for (const slice of fixture.slices) { const relativePath = `slices/${slice.name}`; const bytes = await readFile(join(extracted.output_root, relativePath)); - expect(bytes).toEqual(Buffer.from(slice.bytes)); + expect(Buffer.compare(bytes, Buffer.from(slice.bytes))).toBe(0); expect( extracted.artifacts.find( ({ relative_path }) => relative_path === relativePath, @@ -107,6 +107,6 @@ export const expectArmSliceExtraction = async ( bytes_written: slice.bytes.length, }); } - expect(await readFile(fixture.path)).toEqual(fixture.bytes); + expect(Buffer.compare(await readFile(fixture.path), fixture.bytes)).toBe(0); return extracted.manifest; }; diff --git a/tests/process-global/artifactFileSourceLifecycle.test.ts b/tests/process-global/artifactFileSourceLifecycle.test.ts index e06e680368..4f23fd690e 100644 --- a/tests/process-global/artifactFileSourceLifecycle.test.ts +++ b/tests/process-global/artifactFileSourceLifecycle.test.ts @@ -264,7 +264,7 @@ it.each(carriers)( await reader.close(); expect(close).not.toHaveBeenCalled(); expect(handle.fd).toBeGreaterThanOrEqual(0); - expect(await readFile(path)).toEqual(replacementBytes); + expect(Buffer.compare(await readFile(path), replacementBytes)).toBe(0); expect(control.handles.size).toBe(1); await handle.close(); expect(close).toHaveBeenCalledOnce(); diff --git a/tests/process-global/asarSnapshotOperation.test.ts b/tests/process-global/asarSnapshotOperation.test.ts index eaaee96865..02fb11b0ab 100644 --- a/tests/process-global/asarSnapshotOperation.test.ts +++ b/tests/process-global/asarSnapshotOperation.test.ts @@ -40,7 +40,9 @@ it("serializes concurrent ASAR snapshot preparation and cleans its single owned preparingSnapshot, ]); if (firstEntry.done) throw new Error("Expected an ASAR member"); - expect(await buffer(snapshot)).toEqual(await readFile(archive)); + expect( + Buffer.compare(await buffer(snapshot), await readFile(archive)), + ).toBe(0); const mismatch = reader.prepareContainer("0".repeat(64)); const matching = reader.prepareContainer(expectedDigest); diff --git a/tests/process-global/cutterDescriptorLifetime.test.ts b/tests/process-global/cutterDescriptorLifetime.test.ts index 86e4091e95..3e4e87d0f0 100644 --- a/tests/process-global/cutterDescriptorLifetime.test.ts +++ b/tests/process-global/cutterDescriptorLifetime.test.ts @@ -255,9 +255,10 @@ it("does not assign a peer's later cancellation to a descriptor that already fai error.cause.errors[0] instanceof AnalysisCancelledError, ); expect(cancelled).toHaveLength(1); - expect((cancelled[0]?.cause as AggregateError).errors[0].cause).toBe( - reason, - ); + const cancelledCause = cancelled[0]?.cause; + if (!(cancelledCause instanceof AggregateError)) + throw new Error("Expected descriptor cancellation failure"); + expect(cancelledCause.errors[0].cause).toBe(reason); expect(failure.cleanupIncomplete).toBe(true); fault.blocked.clear(); await client.close(); diff --git a/tests/process-global/ghidraImportSnapshotLifetime.test.ts b/tests/process-global/ghidraImportSnapshotLifetime.test.ts index 66160bab60..effa887bcd 100644 --- a/tests/process-global/ghidraImportSnapshotLifetime.test.ts +++ b/tests/process-global/ghidraImportSnapshotLifetime.test.ts @@ -381,7 +381,7 @@ it("copies every byte across chunk boundaries through the destination owner", as const snapshot = await createGhidraTargetSnapshot(path, runtime, sha256, { platform: "linux", }); - expect(await readFile(snapshot.path)).toEqual(bytes); + expect(Buffer.compare(await readFile(snapshot.path), bytes)).toBe(0); expect(snapshot.sha256).toBe(sha256); expect( [...fault.handles, ...fault.destinations].every( diff --git a/tests/process-global/mainStartupCancellation.test.ts b/tests/process-global/mainStartupCancellation.test.ts index d37d326626..e352250677 100644 --- a/tests/process-global/mainStartupCancellation.test.ts +++ b/tests/process-global/mainStartupCancellation.test.ts @@ -222,7 +222,9 @@ it("hands exact startup bytes to the installed SDK framing owner", async () => { owner.resume(); input.write(ping.subarray(11)); await expect.poll(() => messages.length).toBe(2); - expect(Buffer.concat(raw)).toEqual(Buffer.concat([request, ping])); + expect( + Buffer.compare(Buffer.concat(raw), Buffer.concat([request, ping])), + ).toBe(0); expect(messages[0]).toMatchObject({ params: { clientInfo: { name: "é🚀" } }, }); diff --git a/tests/process-global/regularFileNoFollowFallback.test.ts b/tests/process-global/regularFileNoFollowFallback.test.ts index 0e6e549bbd..c32d7ef4f1 100644 --- a/tests/process-global/regularFileNoFollowFallback.test.ts +++ b/tests/process-global/regularFileNoFollowFallback.test.ts @@ -50,7 +50,9 @@ it("rejects a symlink when O_NOFOLLOW is unavailable and still follows when requ symlinks: "follow", }); try { - await expect(readFile(followed)).resolves.toEqual(await readFile(target)); + expect( + Buffer.compare(await readFile(followed), await readFile(target)), + ).toBe(0); } finally { await followed.close(); } diff --git a/tests/process-global/setupConfigurationConflicts.test.ts b/tests/process-global/setupConfigurationConflicts.test.ts index a2e35e6b31..f480181362 100644 --- a/tests/process-global/setupConfigurationConflicts.test.ts +++ b/tests/process-global/setupConfigurationConflicts.test.ts @@ -111,7 +111,7 @@ it.each(["before", "after"] as const)( format: "json", }), ).toMatchObject({ status: "failed", reason: "conflict" }); - expect(await readFile(path)).toEqual(changed); + expect(Buffer.compare(await readFile(path), changed)).toBe(0); expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(original); expect(await readdir(root)).toEqual([ "config.json", From ed86f9c3bb7910fb8e35cd7a3df3a50a50da58a8 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:53:17 +0800 Subject: [PATCH 30/37] fix(verifiers): propagate session assertion failures synchronously Remove the redundant async declaration from the Ghidra session assertion helper so its existing callers receive assertion failures within their synchronous error boundary. Keep call order unchanged. Validation: a public-helper Node subprocess accepts authored valid handshake facts and rejects incomplete analysis and a target-digest mismatch. The regression failed for the original async declaration with Missing expected exception, then passed with the fix. No installed Ghidra execution was performed. --- scripts/verify-real-ghidra-assertions.mjs | 6 +- .../process/ghidraVerifierAssertions.test.ts | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 tests/boundary/process/ghidraVerifierAssertions.test.ts diff --git a/scripts/verify-real-ghidra-assertions.mjs b/scripts/verify-real-ghidra-assertions.mjs index ae9dbbbbc3..f79f287df9 100644 --- a/scripts/verify-real-ghidra-assertions.mjs +++ b/scripts/verify-real-ghidra-assertions.mjs @@ -245,11 +245,7 @@ export function dossierSummary(dossier) { }; } -export async function assertSession( - session, - expectedDigest, - expectedTargetSha256, -) { +export function assertSession(session, expectedDigest, expectedTargetSha256) { if ( session.provider.id !== "ghidra" || session.provider.version !== SUPPORTED_GHIDRA_VERSION || diff --git a/tests/boundary/process/ghidraVerifierAssertions.test.ts b/tests/boundary/process/ghidraVerifierAssertions.test.ts new file mode 100644 index 0000000000..bd6554bf7e --- /dev/null +++ b/tests/boundary/process/ghidraVerifierAssertions.test.ts @@ -0,0 +1,68 @@ +import { expect } from "vitest"; + +import { processTest } from "../../support/process/processFixture.js"; + +processTest( + "Ghidra verifier session assertions fail within the caller's synchronous error boundary", + async ({ processes }) => { + const assertionsUrl = new URL( + "../../../scripts/verify-real-ghidra-assertions.mjs", + import.meta.url, + ).href; + const valuesUrl = new URL( + "../../../dist/ghidra/GhidraSessionValues.js", + import.meta.url, + ).href; + const policyUrl = new URL( + "../../../dist/ghidra/GhidraReleasePolicy.js", + import.meta.url, + ).href; + // Authored handshake facts exercise the verifier, not a running provider. + const source = ` + import assert from "node:assert/strict"; + import { assertSession } from ${JSON.stringify(assertionsUrl)}; + import { GHIDRA_SESSION_CAPABILITIES } from ${JSON.stringify(valuesUrl)}; + import { SUPPORTED_GHIDRA_VERSION } from ${JSON.stringify(policyUrl)}; + + const profileDigest = "a".repeat(64); + const targetSha256 = "b".repeat(64); + const session = { + name: "REA Ghidra bridge", + run_id: "d6fcbb66-e829-4ff6-a535-0035aec63139", + profile_digest: profileDigest, + provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION }, + read_only: false, + analysis_complete: true, + analysis_timed_out: false, + capabilities: [...GHIDRA_SESSION_CAPABILITIES], + target: { + name: "fixture.bin", + language_id: "x86:LE:64:default", + compiler_spec_id: "gcc", + image_base: "0x401000", + default_address_space: "ram", + sha256: targetSha256, + }, + }; + + assert.doesNotThrow(() => assertSession(session, profileDigest, targetSha256)); + assert.throws( + () => assertSession( + { ...session, analysis_complete: false }, profileDigest, targetSha256, + ), + /Ghidra session commitment drifted/, + ); + assert.throws( + () => assertSession(session, profileDigest, "c".repeat(64)), + /Ghidra session commitment drifted/, + ); + `; + const result = await processes.run( + process.execPath, + ["--input-type=module", "--eval", source], + { timeoutMs: 10_000 }, + ); + expect(result.signal, result.stderr).toBeNull(); + expect(result.exitCode, result.stderr).toBe(0); + }, +); From 76afdd35ae51d0c3e6ce4309a6985195463d4201 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:56:15 +0800 Subject: [PATCH 31/37] chore(lint): guard validation ownership and asynchronous assertions --- .oxlintrc.json | 72 ++++- docs/testing.md | 32 ++- knip.production.json | 35 +++ licenses/npm-dependencies.txt | 37 ++- package-lock.json | 103 +++++++ package.json | 11 +- scripts/ci/plan.mjs | 2 +- .../ci/production-reachability-report.d.mts | 33 +++ scripts/ci/production-reachability-report.mjs | 86 ++++++ scripts/ci/static-check-report.mjs | 46 ++- scripts/lint/index.mjs | 7 + scripts/lint/rules/import-source.d.mts | 4 + scripts/lint/rules/import-source.mjs | 20 ++ .../rules/no-evidence-json-preadmission.d.mts | 4 + .../rules/no-evidence-json-preadmission.mjs | 137 +++++++++ .../lint/rules/no-fixture-root-cleanup.d.mts | 3 + .../lint/rules/no-fixture-root-cleanup.mjs | 149 ++++++++++ .../rules/no-positional-tool-contract.d.mts | 3 + .../rules/no-positional-tool-contract.mjs | 218 ++++++++++++++ scripts/lint/rules/tool-catalog-sources.d.mts | 4 + scripts/lint/rules/tool-catalog-sources.mjs | 118 ++++++++ ...ctionObligationCandidates.semantic.test.ts | 3 +- ...ReconstructionObligationCandidates.test.ts | 13 +- .../ArtifactOccurrenceIdentity.test.ts | 5 +- .../PyInstallerFormatBoundaries.test.ts | 5 +- .../PlaywrightElectronActiveActions.ts | 1 - src/domain/apple/appleApplication.test.ts | 5 +- src/domain/artifactInventoryEvidence.test.ts | 7 +- src/domain/changedBehavior.test.ts | 14 +- src/domain/functionComparison.test.ts | 2 +- src/domain/staticRuntimeCorrelation.test.ts | 14 +- src/domain/webMcpDiscovery.test.ts | 3 +- .../artifactComparisonCoverage.test.ts | 4 +- .../browser/cdpCleanupCancellation.test.ts | 5 +- .../cdpProviderCleanupOwnership.test.ts | 3 +- .../filesystem/artifactComparison.test.ts | 28 +- .../artifactComparisonSummary.test.ts | 4 +- .../artifactDirectoryIdentity.test.ts | 4 +- .../boundary/filesystem/ciChangeScope.test.ts | 24 ++ .../javascriptApplicationWorkflows.test.ts | 8 +- .../filesystem/lintRuleConfiguration.test.ts | 270 ++++++++++++++++++ .../productionReachabilityReport.test.ts | 117 ++++++++ .../filesystem/staticCheckReports.test.ts | 15 +- .../filesystem/typedLintRules.test.ts | 192 +++++++++++++ .../webArtifactReadOwnership.test.ts | 5 +- .../mcp/artifactMcp.inlineInventory.test.ts | 6 +- tests/boundary/mcp/artifactMcp.test.ts | 8 +- .../boundary/mcp/evidenceBundleInline.test.ts | 4 +- .../functionComparisonIntegrityMcp.test.ts | 4 +- .../mcp/functionComparisonMcp.test.ts | 7 +- ...unctionComparisonReferenceFactsMcp.test.ts | 6 +- ...unctionComparisonStringDecodingMcp.test.ts | 6 +- .../mcp/investigationAnalysisMcp.test.ts | 14 +- .../managedWorkflowUnknownIdentityMcp.test.ts | 12 +- .../processComparisonDiagnosticsMcp.test.ts | 12 +- .../processComparisonVerificationMcp.test.ts | 6 +- .../mcp/processTraceAdaptersParity.test.ts | 4 +- tests/boundary/mcp/promptCompletion.test.ts | 6 +- tests/boundary/mcp/serverUnknownsMcp.test.ts | 4 +- .../mcp/workflowUnknownIdentityMcp.test.ts | 12 +- .../managed/managedMemberComparison.test.ts | 24 +- .../evidenceJsonPreadmissionRule.test.ts | 265 +++++++++++++++++ .../evaluation/fixtureRootCleanupRule.test.ts | 58 ++++ .../positionalToolContractRule.test.ts | 184 ++++++++++++ tests/fixtures/artifactEvidence.ts | 5 +- tests/fixtures/nativeFunctionMatching.ts | 6 +- tests/fixtures/oxlintConfiguration.ts | 39 +++ .../reconstructionProofApplicability.ts | 4 +- tests/fixtures/webScriptCapture.ts | 3 +- 69 files changed, 2384 insertions(+), 190 deletions(-) create mode 100644 knip.production.json create mode 100644 scripts/ci/production-reachability-report.d.mts create mode 100644 scripts/ci/production-reachability-report.mjs create mode 100644 scripts/lint/rules/import-source.d.mts create mode 100644 scripts/lint/rules/import-source.mjs create mode 100644 scripts/lint/rules/no-evidence-json-preadmission.d.mts create mode 100644 scripts/lint/rules/no-evidence-json-preadmission.mjs create mode 100644 scripts/lint/rules/no-fixture-root-cleanup.d.mts create mode 100644 scripts/lint/rules/no-fixture-root-cleanup.mjs create mode 100644 scripts/lint/rules/no-positional-tool-contract.d.mts create mode 100644 scripts/lint/rules/no-positional-tool-contract.mjs create mode 100644 scripts/lint/rules/tool-catalog-sources.d.mts create mode 100644 scripts/lint/rules/tool-catalog-sources.mjs create mode 100644 tests/boundary/filesystem/lintRuleConfiguration.test.ts create mode 100644 tests/boundary/filesystem/productionReachabilityReport.test.ts create mode 100644 tests/boundary/filesystem/typedLintRules.test.ts create mode 100644 tests/evaluation/evidenceJsonPreadmissionRule.test.ts create mode 100644 tests/evaluation/fixtureRootCleanupRule.test.ts create mode 100644 tests/evaluation/positionalToolContractRule.test.ts create mode 100644 tests/fixtures/oxlintConfiguration.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index 0a760d3d6b..1a8119d65e 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -1,7 +1,6 @@ { "$schema": "./node_modules/oxlint/configuration_schema.json", - "plugins": ["typescript", "import", "unicorn", "oxc"], - "jsPlugins": [{ "name": "rea", "specifier": "./scripts/lint/index.mjs" }], + "plugins": ["typescript", "import", "unicorn", "oxc", "vitest"], "rules": { "rea/no-buffer-deep-equality": "error", "rea/cancellation-cause": "error", @@ -29,7 +28,37 @@ "max-params": ["warn", 4], "no-var": "warn", "no-var-requires": "error", - "typescript/no-import-type-side-effects": "error" + "typescript/no-import-type-side-effects": "error", + "vitest/no-standalone-expect": "off", + "vitest/no-conditional-expect": "off", + "vitest/expect-expect": "off", + "vitest/no-disabled-tests": "off", + "vitest/require-mock-type-parameters": "off", + "vitest/require-to-throw-message": "off", + "import/no-duplicates": [ + "warn", + { + "preferInline": false + } + ], + "vitest/valid-expect": [ + "warn", + { + "maxArgs": 2 + } + ], + "rea/no-evidence-json-preadmission": "error", + "rea/no-positional-tool-contract": "error", + "rea/no-fixture-root-cleanup": "error", + "typescript/no-floating-promises": "warn", + "typescript/no-misused-promises": "warn", + "typescript/return-await": ["warn", "error-handling-correctness-only"], + "typescript/switch-exhaustiveness-check": [ + "warn", + { + "considerDefaultExhaustiveForUnions": false + } + ] }, "overrides": [ { @@ -42,6 +71,12 @@ "no-var-requires": "off" } }, + { + "files": ["src/**/*.ts"], + "rules": { + "typescript/no-misused-promises": "error" + } + }, { "files": ["tests/**/*.ts", "src/**/*.test.ts", "src/**/*.fixture.ts"], "rules": { @@ -53,7 +88,8 @@ "skipBlankLines": true, "skipComments": true } - ] + ], + "typescript/no-misused-promises": "warn" } }, { @@ -97,6 +133,25 @@ } ] } + }, + { + "files": ["tests/**/*.test.ts", "src/**/*.test.ts"], + "rules": { + "vitest/valid-expect": [ + "warn", + { + "maxArgs": 2 + } + ], + "vitest/valid-expect-in-promise": "error", + "vitest/no-focused-tests": "error" + } + }, + { + "files": ["src/**/*.test.ts"], + "rules": { + "typescript/no-floating-promises": "error" + } } ], "env": { @@ -115,5 +170,14 @@ ".devin", ".factory", "third_party" + ], + "options": { + "reportUnusedDisableDirectives": "error" + }, + "jsPlugins": [ + { + "name": "rea", + "specifier": "./scripts/lint/index.mjs" + } ] } diff --git a/docs/testing.md b/docs/testing.md index 7af365f6f2..f9dc40c84a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -159,13 +159,31 @@ cause when a directly checked caller `AbortSignal` is converted to an follow indirect cancellation helpers, or prove that an explicit cause equals the caller's reason. Keep focused boundary and lifecycle tests for those facts. -Type-aware linting remains outside the routine gate: a full source, test, and -script scan exceeded the bounded process-family memory envelope, including -with Go runtime limits. Future scoped evaluation can consider -`typescript/no-floating-promises`, `typescript/return-await` with -`error-handling-correctness-only`, `typescript/no-misused-promises`, and -`typescript/switch-exhaustiveness-check`. None replaces behavioral checks for -source identity, omission versus unknown, or Evidence provenance. +`rea/no-evidence-json-preadmission` rejects direct generic JSON parses in +`createEvidence` payload expressions; it does not analyze intermediate values. +`rea/no-positional-tool-contract` rejects catalog indices used to select +operation names, schemas or types; display-only selection remains allowed. +`rea/no-fixture-root-cleanup` follows `const` fixture-root aliases into teardown +deletion; mutable bindings and mixed root arrays remain outside its scope. + +Focused `.only` declarations, broken assertion promise chains, and unused disable +directives are errors. `vitest/valid-expect` warns (`maxArgs: 2`) because valid stored and +collected assertions can be awaited later. `import/no-duplicates` warns with +`preferInline: false`, preserving separate type imports. + +Fast lint stays syntax-based. `npm run lint:typed` runs serially with pinned +`oxlint-tsgolint` 7.0.2005 and records native plus typed diagnostics in +`.cache/static-checks/typed-oxlint.json`. Production `src` treats +`no-misused-promises` as an error; tests/fixtures warn. `no-floating-promises` is +an error in `src/**/*.test.ts`; other scopes, return-await and exhaustiveness +findings warn pending triage. + +`npm run knip:production` writes advisory candidates to +`.cache/static-checks/production-reachability.json`; execution/report errors fail. +Ordinary Knip remains. `check`, `check:pr`, and `check:ci:deep` run reachability +then typed lint sequentially after cached work. Typed lint builds the project +type graph and stays separate from fast feedback. Its default Go settings are +`GOMAXPROCS=2` and `GOMEMLIMIT=1024MiB`; caller-selected settings take precedence. Routine iterations and rebases need focused regressions and relevant checks. Before handing off a PR, run `npm run check` and generated-document checks when diff --git a/knip.production.json b/knip.production.json new file mode 100644 index 0000000000..5ccf4fd2ad --- /dev/null +++ b/knip.production.json @@ -0,0 +1,35 @@ +{ + "$schema": "./node_modules/knip/schema.json", + "entry": [ + "src/main.ts!", + "src/cli.ts!", + "src/cliOutput.ts!", + "src/cliBuiltinArguments.ts!", + "src/cli/cliResultOutput.ts!", + "src/mcpDoctor.ts!", + "src/composition/javascriptAnalysisWorkerProcess.ts!", + "src/composition/webBundleAnalysisWorkerProcess.ts!", + "src/sqlite/SqliteDatabaseWorker.ts!", + "src/evm/EvmoleInterfaceWorker.ts!", + "src/browser/history/NodeCaptureProcess.ts!", + "src/javascript/sourceMaps/SourceMapCodecProcess.ts!", + "scripts/rea.mjs!", + "scripts/rea-android-workspace-guardian.mjs!", + "scripts/electron-active-hook.cjs!", + "scripts/prepack.mjs!", + "scripts/prepare.mjs!", + "scripts/verify-windows-native-artifact.mjs!" + ], + "project": ["src/**/*.ts!", "!src/**/*.test.ts", "!src/**/*.fixture.ts"], + "ignoreExportsUsedInFile": true, + "include": [ + "files", + "exports", + "nsExports", + "types", + "nsTypes", + "enumMembers", + "namespaceMembers", + "duplicates" + ] +} diff --git a/licenses/npm-dependencies.txt b/licenses/npm-dependencies.txt index aaf5608cba..df798b062d 100644 --- a/licenses/npm-dependencies.txt +++ b/licenses/npm-dependencies.txt @@ -2,7 +2,7 @@ REA npm dependency inventory Prepared from the root package-lock.json; refresh this inventory when locked dependencies change. License identifiers are upstream lockfile declarations, not a legal assessment. Runtime includes transitive and optional platform packages; development includes build, test and website tooling. -Locked package entries: 956; runtime: 180. +Locked package entries: 963; runtime: 180. Full runtime copyright/license/NOTICE texts: npm-runtime-notices.txt. @algolia/abtesting@1.25.0 | development | MIT @@ -1100,6 +1100,36 @@ commander@13.1.0 | runtime | MIT Package: https://www.npmjs.com/package/@oxfmt/binding-win32-x64-msvc/v/0.72.0 Archive: https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.72.0.tgz Integrity: sha512-uhooIN+DCzWX5uH7mQkSVCsODqsGluGJ4n3ch4+mITtWKpESMDjUJUSDBAVFuYQDwqY2McPDKdKJbM0oZVW/Qw== +@oxlint-tsgolint/darwin-arm64@7.0.2005 | development, optional | MIT + Lockfile path: node_modules/@oxlint-tsgolint/darwin-arm64 + Package: https://www.npmjs.com/package/@oxlint-tsgolint/darwin-arm64/v/7.0.2005 + Archive: https://registry.npmjs.org/@oxlint-tsgolint/darwin-arm64/-/darwin-arm64-7.0.2005.tgz + Integrity: sha512-VqDZxcZVjlonNLuuvraVNgDQt+1ndwa4yJo8OqsK4HYh/61ba9IlMStCVfi/aaIqRoDhNbAKC8gdcTrXqnb+Ug== +@oxlint-tsgolint/darwin-x64@7.0.2005 | development, optional | MIT + Lockfile path: node_modules/@oxlint-tsgolint/darwin-x64 + Package: https://www.npmjs.com/package/@oxlint-tsgolint/darwin-x64/v/7.0.2005 + Archive: https://registry.npmjs.org/@oxlint-tsgolint/darwin-x64/-/darwin-x64-7.0.2005.tgz + Integrity: sha512-4kUfLRNYoVpv0Tm3jPYrvxyfOc/LmdvFC3pKZ4tL6K7lA7K/VS81SAtmxRzbYXYSyb3SBWhwsuX+df4neqzxxA== +@oxlint-tsgolint/linux-arm64@7.0.2005 | development, optional | MIT + Lockfile path: node_modules/@oxlint-tsgolint/linux-arm64 + Package: https://www.npmjs.com/package/@oxlint-tsgolint/linux-arm64/v/7.0.2005 + Archive: https://registry.npmjs.org/@oxlint-tsgolint/linux-arm64/-/linux-arm64-7.0.2005.tgz + Integrity: sha512-rhUNcPsBlKvhXtrbhbBUidtRBRdJ8fbEHN9H5Q5N59gby18ruGuDA7KUxl7oRnUzp797uykYQhkBDm+0KpfK/Q== +@oxlint-tsgolint/linux-x64@7.0.2005 | development, optional | MIT + Lockfile path: node_modules/@oxlint-tsgolint/linux-x64 + Package: https://www.npmjs.com/package/@oxlint-tsgolint/linux-x64/v/7.0.2005 + Archive: https://registry.npmjs.org/@oxlint-tsgolint/linux-x64/-/linux-x64-7.0.2005.tgz + Integrity: sha512-tKs9yinDQZOHygM0e8EJnjTU3mzJtdfkOZP6Udu82iLMePUSHOhTJ9zzU/mSd1hQxURI4LTRRiFhfzC+PzwdNA== +@oxlint-tsgolint/win32-arm64@7.0.2005 | development, optional | MIT + Lockfile path: node_modules/@oxlint-tsgolint/win32-arm64 + Package: https://www.npmjs.com/package/@oxlint-tsgolint/win32-arm64/v/7.0.2005 + Archive: https://registry.npmjs.org/@oxlint-tsgolint/win32-arm64/-/win32-arm64-7.0.2005.tgz + Integrity: sha512-VYPnu1gjfbSv+ROhuyNfMs382D4zQU2ZRc8Uc3nuEZo1k2kf+1KfTrg4hrlWh73h9GI+dUWyo+DYCQzUBlPwcQ== +@oxlint-tsgolint/win32-x64@7.0.2005 | development, optional | MIT + Lockfile path: node_modules/@oxlint-tsgolint/win32-x64 + Package: https://www.npmjs.com/package/@oxlint-tsgolint/win32-x64/v/7.0.2005 + Archive: https://registry.npmjs.org/@oxlint-tsgolint/win32-x64/-/win32-x64-7.0.2005.tgz + Integrity: sha512-ekU89cTGy9fpQzuvxIsFHrnoP8XoFjGIRgPEW6eqgY2NYjGy9pwpjF79o04WXRMJFKRrcaVqkZXUUBdLsqpQDw== @oxlint/binding-android-arm-eabi@1.87.0 | development, optional | MIT Lockfile path: node_modules/@oxlint/binding-android-arm-eabi Package: https://www.npmjs.com/package/@oxlint/binding-android-arm-eabi/v/1.87.0 @@ -3540,6 +3570,11 @@ oxlint@1.87.0 | development | MIT Package: https://www.npmjs.com/package/oxlint/v/1.87.0 Archive: https://registry.npmjs.org/oxlint/-/oxlint-1.87.0.tgz Integrity: sha512-drxVlaW7Oqq7wSy1CN25znRu00aRq8n57Q2wniJRxCmtYVz07ufCm7EVDPQFZURrTgQIjzKrVEHAD/HcvJXObg== +oxlint-tsgolint@7.0.2005 | development | MIT + Lockfile path: node_modules/oxlint-tsgolint + Package: https://www.npmjs.com/package/oxlint-tsgolint/v/7.0.2005 + Archive: https://registry.npmjs.org/oxlint-tsgolint/-/oxlint-tsgolint-7.0.2005.tgz + Integrity: sha512-6ccPGu4u4J9bwYPHXBFPee3RZrX3BboRdU7k14zRDDB1lvN3O7J/vPRFdaEfa1+CWx4xnQQOp4ctoaF66oO0uw== p-limit@2.3.0 | development | MIT Lockfile path: node_modules/p-limit Package: https://www.npmjs.com/package/p-limit/v/2.3.0 diff --git a/package-lock.json b/package-lock.json index 170aee203d..0efee88df8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -73,6 +73,7 @@ "lint-staged": "16.4.0", "oxfmt": "0.72.0", "oxlint": "1.87.0", + "oxlint-tsgolint": "7.0.2005", "release-please": "17.3.0", "rolldown": "1.2.12", "tsrs": "0.18.0-ts7.1.0-dev.20260929", @@ -3491,6 +3492,90 @@ "node": "^20.19.0 || >=22.12.0" } }, + "node_modules/@oxlint-tsgolint/darwin-arm64": { + "version": "7.0.2005", + "resolved": "https://registry.npmjs.org/@oxlint-tsgolint/darwin-arm64/-/darwin-arm64-7.0.2005.tgz", + "integrity": "sha512-VqDZxcZVjlonNLuuvraVNgDQt+1ndwa4yJo8OqsK4HYh/61ba9IlMStCVfi/aaIqRoDhNbAKC8gdcTrXqnb+Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@oxlint-tsgolint/darwin-x64": { + "version": "7.0.2005", + "resolved": "https://registry.npmjs.org/@oxlint-tsgolint/darwin-x64/-/darwin-x64-7.0.2005.tgz", + "integrity": "sha512-4kUfLRNYoVpv0Tm3jPYrvxyfOc/LmdvFC3pKZ4tL6K7lA7K/VS81SAtmxRzbYXYSyb3SBWhwsuX+df4neqzxxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@oxlint-tsgolint/linux-arm64": { + "version": "7.0.2005", + "resolved": "https://registry.npmjs.org/@oxlint-tsgolint/linux-arm64/-/linux-arm64-7.0.2005.tgz", + "integrity": "sha512-rhUNcPsBlKvhXtrbhbBUidtRBRdJ8fbEHN9H5Q5N59gby18ruGuDA7KUxl7oRnUzp797uykYQhkBDm+0KpfK/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxlint-tsgolint/linux-x64": { + "version": "7.0.2005", + "resolved": "https://registry.npmjs.org/@oxlint-tsgolint/linux-x64/-/linux-x64-7.0.2005.tgz", + "integrity": "sha512-tKs9yinDQZOHygM0e8EJnjTU3mzJtdfkOZP6Udu82iLMePUSHOhTJ9zzU/mSd1hQxURI4LTRRiFhfzC+PzwdNA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxlint-tsgolint/win32-arm64": { + "version": "7.0.2005", + "resolved": "https://registry.npmjs.org/@oxlint-tsgolint/win32-arm64/-/win32-arm64-7.0.2005.tgz", + "integrity": "sha512-VYPnu1gjfbSv+ROhuyNfMs382D4zQU2ZRc8Uc3nuEZo1k2kf+1KfTrg4hrlWh73h9GI+dUWyo+DYCQzUBlPwcQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@oxlint-tsgolint/win32-x64": { + "version": "7.0.2005", + "resolved": "https://registry.npmjs.org/@oxlint-tsgolint/win32-x64/-/win32-x64-7.0.2005.tgz", + "integrity": "sha512-ekU89cTGy9fpQzuvxIsFHrnoP8XoFjGIRgPEW6eqgY2NYjGy9pwpjF79o04WXRMJFKRrcaVqkZXUUBdLsqpQDw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@oxlint/binding-android-arm-eabi": { "version": "1.87.0", "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.87.0.tgz", @@ -10137,6 +10222,24 @@ } } }, + "node_modules/oxlint-tsgolint": { + "version": "7.0.2005", + "resolved": "https://registry.npmjs.org/oxlint-tsgolint/-/oxlint-tsgolint-7.0.2005.tgz", + "integrity": "sha512-6ccPGu4u4J9bwYPHXBFPee3RZrX3BboRdU7k14zRDDB1lvN3O7J/vPRFdaEfa1+CWx4xnQQOp4ctoaF66oO0uw==", + "dev": true, + "license": "MIT", + "bin": { + "tsgolint": "bin/tsgolint.js" + }, + "optionalDependencies": { + "@oxlint-tsgolint/darwin-arm64": "7.0.2005", + "@oxlint-tsgolint/darwin-x64": "7.0.2005", + "@oxlint-tsgolint/linux-arm64": "7.0.2005", + "@oxlint-tsgolint/linux-x64": "7.0.2005", + "@oxlint-tsgolint/win32-arm64": "7.0.2005", + "@oxlint-tsgolint/win32-x64": "7.0.2005" + } + }, "node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", diff --git a/package.json b/package.json index 28356cfa87..a1e8ae5d6c 100644 --- a/package.json +++ b/package.json @@ -112,7 +112,7 @@ "prebuild": "npm run deps:check", "build": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run artifacts:skills --force", "precheck": "npm run deps:check", - "check": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check && npm run verify:architecture-guards", + "check": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check && npm run verify:architecture-guards && npm run knip:production && npm run lint:typed", "precheck:fast": "npm run deps:check", "check:fast": "node scripts/run-turbo.mjs run typecheck lint --output-logs=errors-only", "precheck:changed": "npm run deps:check", @@ -120,14 +120,14 @@ "precheck:test": "npm run deps:check", "check:test": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check test:run", "precheck:pr": "npm run deps:check", - "check:pr": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos", + "check:pr": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos && npm run knip:production && npm run lint:typed", "precheck:ci": "npm run deps:check", "precheck:ci:fast": "npm run deps:check", "precheck:ci:deep": "npm run deps:check", "check:ci": "npm run check:ci:fast && npm run check:ci:deep", "check:ci:fast": "node scripts/run-exclusive.mjs artifacts npm run check:ci:fast:tasks", "check:ci:fast:tasks": "node scripts/run-turbo.mjs run typecheck lint:built format:check", - "check:ci:deep": "node scripts/run-exclusive.mjs artifacts npm run check:ci:deep:tasks && npm run verify:architecture-guards && npm run verify:test-discovery && npm run test:python", + "check:ci:deep": "node scripts/run-exclusive.mjs artifacts npm run check:ci:deep:tasks && npm run verify:architecture-guards && npm run verify:test-discovery && npm run test:python && npm run knip:production && npm run lint:typed", "check:ci:deep:tasks": "node scripts/run-turbo.mjs run knip metadata:check jscpd scan:todos", "check:ci:tasks": "node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check jscpd scan:todos", "ci:static-context": "node scripts/ci/static-task-context.mjs", @@ -326,7 +326,9 @@ "docs:prepare": "node scripts/generate-package-metadata.mjs && node scripts/run-turbo.mjs run artifacts:product-catalog artifacts:managed-evidence artifacts:error-schema", "verify:web:qlog": "npm run build:cached && node scripts/verify-qlog-captures.mjs", "licenses:generate": "node scripts/generate-third-party-inventory.mjs", - "licenses:check": "node scripts/generate-third-party-inventory.mjs --check" + "licenses:check": "node scripts/generate-third-party-inventory.mjs --check", + "lint:typed": "node scripts/ci/static-check-report.mjs typed-lint", + "knip:production": "node scripts/ci/production-reachability-report.mjs" }, "lint-staged": { "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}": [ @@ -404,6 +406,7 @@ "lint-staged": "16.4.0", "oxfmt": "0.72.0", "oxlint": "1.87.0", + "oxlint-tsgolint": "7.0.2005", "release-please": "17.3.0", "rolldown": "1.2.12", "tsrs": "0.18.0-ts7.1.0-dev.20260929", diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 7865e1157f..a93872181b 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -108,7 +108,7 @@ function classifyPackage(base, head) { continue; } if ( - /^(test(?::|$)|check(?::|$)|lint(?::|$)|format(?::|$)|knip$|jscpd$)/.test( + /^(test(?::|$)|check(?::|$)|lint(?::|$)|format(?::|$)|knip(?::|$)|jscpd$)/.test( script, ) ) diff --git a/scripts/ci/production-reachability-report.d.mts b/scripts/ci/production-reachability-report.d.mts new file mode 100644 index 0000000000..3970ca390f --- /dev/null +++ b/scripts/ci/production-reachability-report.d.mts @@ -0,0 +1,33 @@ +export interface ProductionReachabilityFinding { + readonly name: string; + readonly namespace?: string; + readonly kind?: string; + readonly specifier?: string; + readonly line?: number; + readonly col?: number; + readonly pos?: number; +} + +export interface ProductionReachabilityIssue { + readonly file: string; + readonly owners?: readonly { readonly name: string }[]; + readonly files?: readonly ProductionReachabilityFinding[]; + readonly exports?: readonly ProductionReachabilityFinding[]; + readonly nsExports?: readonly ProductionReachabilityFinding[]; + readonly types?: readonly ProductionReachabilityFinding[]; + readonly nsTypes?: readonly ProductionReachabilityFinding[]; + readonly enumMembers?: readonly ProductionReachabilityFinding[]; + readonly namespaceMembers?: readonly ProductionReachabilityFinding[]; + readonly duplicates?: readonly (readonly ProductionReachabilityFinding[])[]; +} + +export interface ProductionReachabilityReport { + readonly issues: readonly ProductionReachabilityIssue[]; +} + +export function parseProductionReachabilityReport( + text: string, +): ProductionReachabilityReport; +export function recordProductionReachabilityReport( + root?: string, +): Promise; diff --git a/scripts/ci/production-reachability-report.mjs b/scripts/ci/production-reachability-report.mjs new file mode 100644 index 0000000000..8758e6a126 --- /dev/null +++ b/scripts/ci/production-reachability-report.mjs @@ -0,0 +1,86 @@ +import { spawnSync } from "node:child_process"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; + +const require = createRequire(import.meta.url); +const finding = z.object({ + name: z.string(), + namespace: z.string().optional(), + kind: z.string().optional(), + specifier: z.string().optional(), + line: z.number().optional(), + col: z.number().optional(), + pos: z.number().optional(), +}); +const issue = z.strictObject({ + file: z.string(), + owners: z.array(z.object({ name: z.string() })).optional(), + files: z.array(finding).optional(), + exports: z.array(finding).optional(), + nsExports: z.array(finding).optional(), + types: z.array(finding).optional(), + nsTypes: z.array(finding).optional(), + enumMembers: z.array(finding).optional(), + namespaceMembers: z.array(finding).optional(), + duplicates: z.array(z.array(finding)).optional(), +}); +const reportSchema = z.strictObject({ issues: z.array(issue) }); + +/** Admit the installed Knip JSON reporter's candidate records. */ +export const parseProductionReachabilityReport = (text) => + reportSchema.parse(JSON.parse(text)); + +/** Run production reachability without turning candidates into failures. */ +export const recordProductionReachabilityReport = async ( + root = process.cwd(), +) => { + const reportDirectory = join(root, ".cache", "static-checks"); + const reportPath = join(reportDirectory, "production-reachability.json"); + + // Findings are advisory. Execution, configuration and report-admission failures + // still fail, and must never leave an older successful report at this path. + await mkdir(reportDirectory, { recursive: true }); + await rm(reportPath, { force: true }); + const result = spawnSync( + process.execPath, + [ + join(dirname(require.resolve("knip")), "..", "bin", "knip.js"), + "--config", + "knip.production.json", + "--production", + "--reporter", + "json", + "--no-exit-code", + ], + { cwd: root, encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }, + ); + if (result.stderr) process.stderr.write(result.stderr); + if (result.error !== undefined) throw result.error; + if (result.status !== 0) { + if (result.stdout) process.stderr.write(result.stdout); + throw new Error( + result.status === null + ? `Production reachability analysis terminated by ${result.signal}` + : `Production reachability analysis failed with status ${result.status}`, + ); + } + const report = parseProductionReachabilityReport(result.stdout); + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); + console.log( + `Production reachability: ${report.issues.length} files with advisory candidates.`, + ); + console.log(`Full report: ${reportPath}`); + console.log( + "Candidates require runtime, verifier and published-contract review before removal; ordinary Knip remains separate.", + ); + return report; +}; + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) + await recordProductionReachabilityReport(); diff --git a/scripts/ci/static-check-report.mjs b/scripts/ci/static-check-report.mjs index eecaf750af..6654f9a60b 100644 --- a/scripts/ci/static-check-report.mjs +++ b/scripts/ci/static-check-report.mjs @@ -48,7 +48,12 @@ const duplicateReport = z }) .passthrough(); -const run = (packageName, entrypoint, arguments_) => { +const run = ( + packageName, + entrypoint, + arguments_, + environment = process.env, +) => { const executable = join( dirname(require.resolve(`${packageName}/package.json`)), entrypoint, @@ -56,6 +61,7 @@ const run = (packageName, entrypoint, arguments_) => { const result = spawnSync(process.execPath, [executable, ...arguments_], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, + env: environment, }); if (result.stderr) process.stderr.write(result.stderr); if (result.error !== undefined) throw result.error; @@ -66,14 +72,30 @@ const run = (packageName, entrypoint, arguments_) => { try { await mkdir(reportRoot, { recursive: true }); - if (mode === "lint") { - const reportPath = join(reportRoot, "oxlint.json"); + if (mode === "lint" || mode === "typed-lint") { + const typed = mode === "typed-lint"; + const reportPath = join( + reportRoot, + typed ? "typed-oxlint.json" : "oxlint.json", + ); await rm(reportPath, { force: true }); - const result = run("oxlint", "bin/oxlint", [ - ...extraArguments, - "--format", - "json", - ]); + const result = run( + "oxlint", + "bin/oxlint", + [ + ...(typed ? ["--type-aware", "--threads", "1"] : []), + ...extraArguments, + "--format", + "json", + ], + typed + ? { + ...process.env, + GOMAXPROCS: process.env.GOMAXPROCS ?? "2", + GOMEMLIMIT: process.env.GOMEMLIMIT ?? "1024MiB", + } + : process.env, + ); let report; try { report = lintReport.parse(JSON.parse(result.stdout)); @@ -89,7 +111,7 @@ try { ({ severity }) => severity === "error", ); console.log( - `Lint: ${report.number_of_files} files, ${errors.length} errors, ${warnings.length} warnings.`, + `${typed ? "Typed lint" : "Lint"}: ${report.number_of_files} files, ${errors.length} errors, ${warnings.length} warnings.`, ); for (const diagnostic of errors) { const location = diagnostic.labels[0]?.span; @@ -106,7 +128,9 @@ try { ) .slice(0, 8)) console.log(` ${rule}: ${count} warnings`); - console.log(`Full diagnostics: ${reportPath}; npm run lint:details`); + console.log( + `Full diagnostics: ${reportPath}${typed ? "" : "; npm run lint:details"}`, + ); process.exitCode = result.status; } else if (mode === "duplicates") { const output = join(reportRoot, "jscpd"); @@ -143,7 +167,7 @@ try { console.log(`Full diagnostics: ${reportPath}; npm run jscpd:details`); process.exitCode = result.status; } else { - throw new Error("Expected lint or duplicates"); + throw new Error("Expected lint, typed-lint or duplicates"); } } catch (cause) { console.error(cause instanceof Error ? cause.message : String(cause)); diff --git a/scripts/lint/index.mjs b/scripts/lint/index.mjs index c04cbd95c4..db0365acfd 100644 --- a/scripts/lint/index.mjs +++ b/scripts/lint/index.mjs @@ -1,10 +1,17 @@ import noBufferDeepEquality from "./no-buffer-deep-equality.mjs"; import cancellationCause from "./cancellation-cause.mjs"; +import noEvidenceJsonPreadmission from "./rules/no-evidence-json-preadmission.mjs"; +import noFixtureRootCleanup from "./rules/no-fixture-root-cleanup.mjs"; +import noPositionalToolContract from "./rules/no-positional-tool-contract.mjs"; + export default { meta: { name: "rea" }, rules: { "no-buffer-deep-equality": noBufferDeepEquality, "cancellation-cause": cancellationCause, + "no-evidence-json-preadmission": noEvidenceJsonPreadmission, + "no-fixture-root-cleanup": noFixtureRootCleanup, + "no-positional-tool-contract": noPositionalToolContract, }, }; diff --git a/scripts/lint/rules/import-source.d.mts b/scripts/lint/rules/import-source.d.mts new file mode 100644 index 0000000000..e7d68389b9 --- /dev/null +++ b/scripts/lint/rules/import-source.d.mts @@ -0,0 +1,4 @@ +export declare const resolveImportSource: ( + filename: string, + specifier: unknown, +) => string | undefined; diff --git a/scripts/lint/rules/import-source.mjs b/scripts/lint/rules/import-source.mjs new file mode 100644 index 0000000000..ec524cefc7 --- /dev/null +++ b/scripts/lint/rules/import-source.mjs @@ -0,0 +1,20 @@ +import { dirname, isAbsolute, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** Resolve local import spellings without assigning local meaning to bare or HTTP imports. */ +export const resolveImportSource = (filename, specifier) => { + if (typeof specifier !== "string") return undefined; + if (isAbsolute(specifier)) return resolve(specifier); + if (specifier.startsWith("file:")) { + try { + const url = new URL(specifier); + if (url.search !== "" || url.hash !== "") return undefined; + return fileURLToPath(url); + } catch { + return undefined; + } + } + if (specifier.startsWith("./") || specifier.startsWith("../")) + return resolve(dirname(filename), specifier); + return undefined; +}; diff --git a/scripts/lint/rules/no-evidence-json-preadmission.d.mts b/scripts/lint/rules/no-evidence-json-preadmission.d.mts new file mode 100644 index 0000000000..86d61729e2 --- /dev/null +++ b/scripts/lint/rules/no-evidence-json-preadmission.d.mts @@ -0,0 +1,4 @@ +import type { RuleTester } from "oxlint/plugins-dev"; + +declare const rule: Parameters[1]; +export default rule; diff --git a/scripts/lint/rules/no-evidence-json-preadmission.mjs b/scripts/lint/rules/no-evidence-json-preadmission.mjs new file mode 100644 index 0000000000..d591d54ad5 --- /dev/null +++ b/scripts/lint/rules/no-evidence-json-preadmission.mjs @@ -0,0 +1,137 @@ +import { findBinding } from "../bindings.mjs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { resolveImportSource } from "./import-source.mjs"; + +const domain = fileURLToPath(new URL("../../../src/domain/", import.meta.url)); +const payloadFields = new Set(["parameters", "result", "rawResult"]); + +const propertyName = (node) => { + if (!node.computed && node.property?.type === "Identifier") + return node.property.name; + if ( + node.property?.type === "Literal" && + typeof node.property.value === "string" + ) + return node.property.value; + return undefined; +}; + +const fieldName = (node) => { + if (node.type !== "Property") return undefined; + if (!node.computed && node.key.type === "Identifier") return node.key.name; + if (node.key.type === "Literal" && typeof node.key.value === "string") + return node.key.value; + return undefined; +}; + +const importedBinding = (sourceCode, node) => { + if (node.type !== "Identifier") return undefined; + const variable = findBinding(sourceCode, node); + if (!variable || variable.defs.length !== 1) return undefined; + const definition = variable.defs[0]; + return definition.type === "ImportBinding" ? definition : undefined; +}; + +const isImported = (context, node, module, names) => { + const member = node.type === "MemberExpression" && !node.optional; + const binding = importedBinding( + context.sourceCode, + member ? node.object : node, + ); + if (!binding || binding.parent?.type !== "ImportDeclaration") return false; + const declaration = binding.parent; + if (declaration.importKind === "type" || binding.node.importKind === "type") + return false; + const specifier = declaration.source.value; + const target = resolveImportSource( + context.physicalFilename ?? context.filename, + specifier, + ); + if ( + target !== resolve(domain, `${module}.js`) && + target !== resolve(domain, `${module}.ts`) + ) + return false; + if (member) + return ( + binding.node.type === "ImportNamespaceSpecifier" && + names.has(propertyName(node)) + ); + if (binding.node.type !== "ImportSpecifier") return false; + return names.has(binding.node.imported.name ?? binding.node.imported.value); +}; + +const evidenceCreator = new Set(["createEvidence"]); +const genericJsonSchemas = new Set(["jsonValueSchema", "jsonObjectSchema"]); + +export default { + meta: { + type: "suggestion", + docs: { + description: + "Leave ordinary Evidence JSON payload admission with createEvidence.", + }, + schema: [], + messages: { + redundantJsonAdmission: + "Pass the {{field}} payload directly to createEvidence; it owns generic JSON admission. Keep semantic validation at its own boundary.", + }, + }, + create(context) { + return { + CallExpression(node) { + if ( + node.optional || + !isImported(context, node.callee, "evidence", evidenceCreator) + ) + return; + const observation = node.arguments[2]; + if (observation?.type !== "ObjectExpression") return; + for (const [index, property] of observation.properties.entries()) { + const field = fieldName(property); + if ( + !payloadFields.has(field) || + property.kind !== "init" || + property.method + ) + continue; + // A later spread or unknown key may replace this field before admission. + if ( + observation.properties + .slice(index + 1) + .some( + (later) => + later.type !== "Property" || + fieldName(later) === undefined || + fieldName(later) === field, + ) + ) + continue; + const parsed = property.value; + if ( + parsed.type !== "CallExpression" || + parsed.optional || + parsed.arguments.length !== 1 + ) + continue; + const parse = parsed.callee; + if ( + parse.type !== "MemberExpression" || + parse.optional || + propertyName(parse) !== "parse" + ) + continue; + if ( + isImported(context, parse.object, "jsonValue", genericJsonSchemas) + ) + context.report({ + node: parsed, + messageId: "redundantJsonAdmission", + data: { field }, + }); + } + }, + }; + }, +}; diff --git a/scripts/lint/rules/no-fixture-root-cleanup.d.mts b/scripts/lint/rules/no-fixture-root-cleanup.d.mts new file mode 100644 index 0000000000..b5656592e8 --- /dev/null +++ b/scripts/lint/rules/no-fixture-root-cleanup.d.mts @@ -0,0 +1,3 @@ +import type { RuleTester } from "oxlint/plugins-dev"; +declare const rule: Parameters[1]; +export default rule; diff --git a/scripts/lint/rules/no-fixture-root-cleanup.mjs b/scripts/lint/rules/no-fixture-root-cleanup.mjs new file mode 100644 index 0000000000..bf97ae1300 --- /dev/null +++ b/scripts/lint/rules/no-fixture-root-cleanup.mjs @@ -0,0 +1,149 @@ +import { findBinding } from "../bindings.mjs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { resolveImportSource } from "./import-source.mjs"; + +const root = fileURLToPath(new URL("../../../", import.meta.url)); +const fixtureModule = resolve(root, "tests/fixtures/temporaryDirectory"); +const workspaceModule = resolve( + root, + "tests/support/workspace/workspaceFixture", +); +const hooks = new Set(["afterEach", "afterAll", "onTestFinished"]); +const unwrap = (node) => { + while ( + node && + [ + "AwaitExpression", + "TSAsExpression", + "TSNonNullExpression", + "ChainExpression", + ].includes(node.type) + ) + node = node.argument ?? node.expression; + return node; +}; +const isFunction = (node) => + node && + [ + "FunctionDeclaration", + "FunctionExpression", + "ArrowFunctionExpression", + ].includes(node.type); + +export default { + meta: { + type: "problem", + docs: { + description: "Let createTestTempDirectory own removal of its root.", + }, + schema: [], + messages: { + duplicate: + "createTestTempDirectory already owns root cleanup; remove the duplicate teardown deletion, retaining cleanup of other resources.", + }, + }, + create(context) { + const sourceCode = context.sourceCode; + const filename = context.physicalFilename ?? context.filename; + const calls = []; + const variable = (node) => findBinding(sourceCode, node); + const imported = (node) => { + const member = node?.type === "MemberExpression" ? node : undefined; + const binding = member ? member.object : node; + if (binding?.type !== "Identifier") return undefined; + const definition = variable(binding)?.defs[0]; + if ( + definition?.type !== "ImportBinding" || + (member + ? definition.node.type !== "ImportNamespaceSpecifier" + : definition.node.type !== "ImportSpecifier") + ) + return undefined; + const declaration = definition.parent ?? definition.node.parent; + const specifier = declaration.source?.value; + if (typeof specifier !== "string") return undefined; + return { + name: member + ? member.computed + ? member.property.value + : member.property.name + : (definition.node.imported.name ?? definition.node.imported.value), + module: + resolveImportSource(filename, specifier)?.replace( + /\.(?:[cm]?[jt]s)$/u, + "", + ) ?? specifier, + }; + }; + const fixtureRoot = (raw, seen = new Set()) => { + const node = unwrap(raw); + if (node?.type === "CallExpression") { + const identity = imported(node.callee); + return ( + identity?.module === fixtureModule && + identity.name === "createTestTempDirectory" + ); + } + if (node?.type !== "Identifier") return false; + const binding = variable(node); + if (!binding || seen.has(binding)) return false; + seen.add(binding); + const definition = binding.defs[0]; + // Mutable ownership and mixed root registries require flow analysis, not + // a guess based on the variable name or one assignment. + return ( + definition?.type === "Variable" && + definition.parent.kind === "const" && + definition.node.id.type === "Identifier" && + fixtureRoot(definition.node.init, seen) + ); + }; + const callback = (raw) => { + const node = unwrap(raw); + if (isFunction(node)) return node; + if (node?.type !== "Identifier") return undefined; + const definition = variable(node)?.defs[0]; + if (definition?.type === "FunctionName") return definition.node; + if ( + definition?.type === "Variable" && + definition.parent.kind === "const" + ) { + const value = unwrap(definition.node.init); + if (isFunction(value)) return value; + } + return undefined; + }; + return { + CallExpression(node) { + calls.push(node); + }, + "Program:exit"() { + const teardowns = new Set(); + for (const call of calls) { + const identity = imported(call.callee); + if (identity?.module === "vitest" && hooks.has(identity.name)) { + const body = callback(call.arguments[0]); + if (body) teardowns.add(body); + } + } + for (const call of calls) { + const identity = imported(call.callee); + if (!identity) continue; + const removesRoot = + (identity.module === "node:fs/promises" && + ["rm", "rmdir"].includes(identity.name)) || + (identity.module === "node:fs" && + ["rm", "rmSync", "rmdir", "rmdirSync"].includes(identity.name)) || + (identity.module === workspaceModule && + identity.name === "removeTestWorkspace"); + if (!removesRoot || !fixtureRoot(call.arguments[0])) continue; + let parent = call.parent; + while (parent && !isFunction(parent)) parent = parent.parent; + if (teardowns.has(parent)) + context.report({ node: call, messageId: "duplicate" }); + } + }, + }; + }, +}; diff --git a/scripts/lint/rules/no-positional-tool-contract.d.mts b/scripts/lint/rules/no-positional-tool-contract.d.mts new file mode 100644 index 0000000000..b5656592e8 --- /dev/null +++ b/scripts/lint/rules/no-positional-tool-contract.d.mts @@ -0,0 +1,3 @@ +import type { RuleTester } from "oxlint/plugins-dev"; +declare const rule: Parameters[1]; +export default rule; diff --git a/scripts/lint/rules/no-positional-tool-contract.mjs b/scripts/lint/rules/no-positional-tool-contract.mjs new file mode 100644 index 0000000000..9a8276db6e --- /dev/null +++ b/scripts/lint/rules/no-positional-tool-contract.mjs @@ -0,0 +1,218 @@ +import { findBinding } from "../bindings.mjs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { collectToolCatalogSources } from "./tool-catalog-sources.mjs"; +import { resolveImportSource } from "./import-source.mjs"; + +const root = fileURLToPath(new URL("../../../", import.meta.url)); +const catalogs = collectToolCatalogSources( + resolve(root, "src/contracts/toolContracts.ts"), +); + +const unwrap = (node) => { + while ( + node && + [ + "TSAsExpression", + "TSSatisfiesExpression", + "TSNonNullExpression", + "ChainExpression", + ].includes(node.type) + ) + node = node.expression; + return node; +}; +const numeric = (node) => + (node?.type === "Literal" && + (typeof node.value === "number" || + (typeof node.value === "string" && + /^(?:0|[1-9][0-9]*)$/u.test(node.value)))) || + (node?.type === "UnaryExpression" && + ["+", "-"].includes(node.operator) && + node.argument.type === "Literal" && + typeof node.argument.value === "number"); + +export default { + meta: { + type: "problem", + docs: { + description: + "Select canonical tool contracts by name, never catalog position.", + }, + schema: [], + messages: { + positional: + "Select a named tool contract; catalog ordering must not select an operation or schema.", + }, + }, + create(context) { + const sourceCode = context.sourceCode; + const filename = context.physicalFilename ?? context.filename; + const variable = (node) => findBinding(sourceCode, node); + const imported = (node) => { + if (node?.type !== "Identifier") return undefined; + const definition = variable(node)?.defs[0]; + if (definition?.type !== "ImportBinding") return undefined; + const declaration = definition.parent ?? definition.node.parent; + const specifier = declaration.source?.value; + const path = resolveImportSource(filename, specifier)?.replace( + /\.(?:[cm]?[jt]s)$/u, + "", + ); + const expected = catalogs.get(path); + if (expected === undefined) return undefined; + return { definition, expected }; + }; + const isCatalog = (raw, seen = new Set()) => { + const node = unwrap(raw); + if (!node) return false; + if (node.type === "Identifier") { + const binding = variable(node); + if (!binding || seen.has(binding)) return false; + seen.add(binding); + const identity = imported(node); + if (identity) { + const specifier = identity.definition.node; + return ( + specifier.type === "ImportSpecifier" && + identity.expected.has( + specifier.imported.name ?? specifier.imported.value, + ) + ); + } + const definition = binding.defs[0]; + return ( + definition?.type === "Variable" && + definition.parent.kind === "const" && + definition.node.id.type === "Identifier" && + isCatalog(definition.node.init, seen) + ); + } + if (node.type === "MemberExpression" || node.type === "TSQualifiedName") { + const object = + node.type === "MemberExpression" ? node.object : node.left; + const identity = imported(object); + if (identity?.definition.node.type !== "ImportNamespaceSpecifier") + return false; + const property = + node.type === "MemberExpression" ? node.property : node.right; + const name = node.computed ? property.value : property.name; + return identity.expected.has(name); + } + return false; + }; + const selectedContract = (raw, seen = new Set()) => { + const node = unwrap(raw); + if (!node) return undefined; + if ( + node.type === "MemberExpression" && + node.computed && + numeric(node.property) && + isCatalog(node.object) + ) + return node; + if (node.type === "CallExpression") { + const callee = unwrap(node.callee); + if ( + callee?.type === "MemberExpression" && + (callee.computed ? callee.property.value : callee.property.name) === + "at" && + numeric(node.arguments[0]) && + isCatalog(callee.object) + ) + return node; + } + if (node.type !== "Identifier") return undefined; + const binding = variable(node); + if (!binding || seen.has(binding)) return undefined; + seen.add(binding); + const definition = binding.defs[0]; + if (definition?.type !== "Variable" || definition.parent.kind !== "const") + return undefined; + if (definition.node.id.type === "Identifier") + return selectedContract(definition.node.init, seen); + if ( + definition.node.id.type === "ArrayPattern" && + definition.node.id.elements.some( + (element) => + element?.type === "Identifier" && element.name === node.name, + ) && + isCatalog(definition.node.init) + ) + return definition.node; + return undefined; + }; + const bindingFields = new Set(["name", "inputSchema", "outputSchema"]); + const isRegistrationOptions = (node) => { + if (node?.type !== "Identifier") return false; + const definition = variable(node)?.defs[0]; + if ( + definition?.type !== "ImportBinding" || + definition.node.type !== "ImportSpecifier" + ) + return false; + const declaration = definition.parent ?? definition.node.parent; + const specifier = declaration.source?.value; + const path = resolveImportSource(filename, specifier)?.replace( + /\.(?:[cm]?[jt]s)$/u, + "", + ); + const name = + definition.node.imported.name ?? definition.node.imported.value; + return ( + path === resolve(root, "src/server/toolRegistrationOptions") && + [ + "toolRegistrationOptions", + "applicationToolRegistrationOptions", + ].includes(name) + ); + }; + const reported = new Set(); + const reportSelection = (raw) => { + const node = selectedContract(raw); + if (node && !reported.has(node)) { + reported.add(node); + context.report({ node, messageId: "positional" }); + } + }; + return { + MemberExpression(node) { + const name = node.computed ? node.property.value : node.property.name; + const parent = node.parent; + const writesProperty = + (parent?.type === "AssignmentExpression" && parent.left === node) || + parent?.type === "UpdateExpression" || + (parent?.type === "UnaryExpression" && parent.operator === "delete"); + if (bindingFields.has(name) && !writesProperty) + reportSelection(node.object); + }, + CallExpression(node) { + if (isRegistrationOptions(unwrap(node.callee))) + reportSelection(node.arguments[0]); + }, + VariableDeclarator(node) { + if ( + node.id.type === "ObjectPattern" && + node.id.properties.some( + (property) => + property.type === "Property" && + bindingFields.has( + property.computed ? property.key.value : property.key.name, + ), + ) + ) + reportSelection(node.init); + }, + TSIndexedAccessType(node) { + if ( + node.objectType.type === "TSTypeQuery" && + node.indexType.type === "TSLiteralType" && + numeric(node.indexType.literal) && + isCatalog(node.objectType.exprName) + ) + context.report({ node, messageId: "positional" }); + }, + }; + }, +}; diff --git a/scripts/lint/rules/tool-catalog-sources.d.mts b/scripts/lint/rules/tool-catalog-sources.d.mts new file mode 100644 index 0000000000..6d45ede868 --- /dev/null +++ b/scripts/lint/rules/tool-catalog-sources.d.mts @@ -0,0 +1,4 @@ +export declare const collectToolCatalogSources: ( + entryPath: string, + readSource?: (path: string) => string, +) => ReadonlyMap>; diff --git a/scripts/lint/rules/tool-catalog-sources.mjs b/scripts/lint/rules/tool-catalog-sources.mjs new file mode 100644 index 0000000000..da07422f86 --- /dev/null +++ b/scripts/lint/rules/tool-catalog-sources.mjs @@ -0,0 +1,118 @@ +import { parse } from "@babel/parser"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +const withoutExtension = (path) => path.replace(/\.(?:[cm]?[jt]s)$/u, ""); +const arrayExpression = (raw) => { + let node = raw; + while ( + node && + ["TSAsExpression", "TSSatisfiesExpression"].includes(node.type) + ) + node = node.expression; + return node; +}; + +/** Read only source composition, never execute the runtime contract catalog. */ +export const collectToolCatalogSources = ( + entryPath, + readSource = (path) => readFileSync(path, "utf8"), +) => { + const catalogs = new Map(); + const modules = new Map(); + const active = new Set(); + const complete = new Set(); + const moduleFacts = (path) => { + let facts = modules.get(path); + if (facts) return facts; + // Read/parse errors intentionally escape: a broken inventory cannot disable + // the rule by presenting an empty set of catalog bindings. + const ast = parse(readSource(path), { + sourceType: "module", + plugins: ["typescript"], + }); + const imports = new Map(); + const exports = new Map(); + const locals = new Map(); + for (const statement of ast.program.body) { + if (statement.type === "ImportDeclaration") { + for (const specifier of statement.specifiers) { + if (specifier.type === "ImportSpecifier") + imports.set(specifier.local.name, { + source: statement.source.value, + name: specifier.imported.name ?? specifier.imported.value, + }); + } + } + const declarations = + statement.type === "ExportNamedDeclaration" + ? statement.declaration + : statement; + if (declarations?.type === "VariableDeclaration") + for (const declaration of declarations.declarations) + if (declaration.id.type === "Identifier") { + locals.set(declaration.id.name, declaration.init); + if (statement.type === "ExportNamedDeclaration") + exports.set(declaration.id.name, declaration.init); + } + } + facts = { imports, exports, locals }; + modules.set(path, facts); + return facts; + }; + const visit = (path, name) => { + const key = JSON.stringify([path, name]); + if (complete.has(key)) return; + if (active.has(key)) + throw new Error(`Cyclic tool catalog composition: ${path}#${name}`); + active.add(key); + const facts = moduleFacts(path); + const array = arrayExpression(facts.exports.get(name)); + if (array?.type !== "ArrayExpression") + throw new Error(`Expected exported tool catalog array: ${path}#${name}`); + const source = withoutExtension(path); + const names = catalogs.get(source) ?? new Set(); + names.add(name); + catalogs.set(source, names); + const composition = (raw, localPath = new Set()) => { + const expression = arrayExpression(raw); + if (expression?.type === "ArrayExpression") { + for (const element of expression.elements) + if (element?.type === "SpreadElement") + composition(element.argument, localPath); + return; + } + // Calls such as HOPPER_MEMORY_TOOL_DEFINITIONS.map(official) construct + // entries locally; they are not imported catalog arrays. Do not execute + // or infer catalog ownership from arbitrary computed expressions. + if (expression?.type !== "Identifier") return; + const binding = facts.imports.get(expression.name); + if (binding) { + if (!binding.source.startsWith(".")) + throw new Error(`Expected local catalog module: ${path}#${name}`); + visit( + `${withoutExtension(resolve(dirname(path), binding.source))}.ts`, + binding.name, + ); + return; + } + if (localPath.has(expression.name)) + throw new Error( + `Cyclic local catalog composition: ${path}#${expression.name}`, + ); + if (!facts.locals.has(expression.name)) + throw new Error( + `Unresolved local catalog spread: ${path}#${expression.name}`, + ); + composition( + facts.locals.get(expression.name), + new Set([...localPath, expression.name]), + ); + }; + composition(array); + active.delete(key); + complete.add(key); + }; + visit(entryPath, "TOOL_CONTRACTS"); + return catalogs; +}; diff --git a/src/application/ReconstructionObligationCandidates.semantic.test.ts b/src/application/ReconstructionObligationCandidates.semantic.test.ts index a56652033f..9b11b56f5f 100644 --- a/src/application/ReconstructionObligationCandidates.semantic.test.ts +++ b/src/application/ReconstructionObligationCandidates.semantic.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from "vitest"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../contracts/javascript/javascriptRuntimeReconciliationExample.js"; import { createEvidence } from "../domain/evidence.js"; import { createEvidenceBundle } from "../domain/evidenceBundle.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js"; import { artifactEvidence } from "../domain/javascript/javascriptApplicationGraph.fixture.js"; import { @@ -108,7 +107,7 @@ const semanticApplication = ( predicateType: example.predicate_type, operation: example.operation, parameters: example.parameters, - result: jsonValueSchema.parse({ ...base, semantic_graph: graph }), + result: { ...base, semantic_graph: graph }, confidence: "derived", authority: "shipped-artifact", }, diff --git a/src/application/ReconstructionObligationCandidates.test.ts b/src/application/ReconstructionObligationCandidates.test.ts index cc15e6b290..759a060e8f 100644 --- a/src/application/ReconstructionObligationCandidates.test.ts +++ b/src/application/ReconstructionObligationCandidates.test.ts @@ -33,7 +33,6 @@ import { createJavaScriptApplicationGraph, createJavaScriptApplicationNode, } from "../domain/javascript/javascriptApplicationGraph.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { MANAGED_WORKFLOW_PROVIDER } from "../domain/investigationProviders.js"; import { createProcessCaptureEvidence } from "./process/ProcessEvidence.js"; @@ -90,7 +89,7 @@ const applicationWithUnknowns = ( predicateType: "rea.managed-application-graph", operation: "project_managed_application_graph", parameters: {}, - result: jsonValueSchema.parse({ + result: { projection_id: `magp_${"a".repeat(64)}`, root_artifact_sha256: APPLICATION_GRAPH_DIGESTS.asar, source_evidence: { @@ -113,7 +112,7 @@ const applicationWithUnknowns = ( coverage: { status: "complete-within-inputs" }, evidence_links: [source.evidence_id], limitations: [], - }), + }, confidence: "inferred", authority: "analyst-inference", evidenceLinks: [source.evidence_id], @@ -212,7 +211,7 @@ describe("reconstruction obligation candidates", () => { predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...rest, manifest, exit: { @@ -224,7 +223,7 @@ describe("reconstruction obligation candidates", () => { ], }, }, - }), + }, }); const generated = deriveCandidates(createEvidenceBundle([evidence]), []); @@ -308,7 +307,7 @@ it("keeps each semantic candidate bound to its own file and authority context", predicateType: example.predicate_type, operation: example.operation, parameters: example.parameters, - result: jsonValueSchema.parse({ ...base, semantic_graph: semanticGraph }), + result: { ...base, semantic_graph: semanticGraph }, confidence: "derived", authority: "shipped-artifact", evidenceLinks: sourceRecords.map(({ evidence_id }) => evidence_id), @@ -415,7 +414,7 @@ it("falls back to a kind-based title when a semantic node label is blank", () => predicateType: example.predicate_type, operation: example.operation, parameters: example.parameters, - result: jsonValueSchema.parse({ ...base, semantic_graph: semanticGraph }), + result: { ...base, semantic_graph: semanticGraph }, confidence: "derived", authority: "shipped-artifact", evidenceLinks: [source.evidence_id], diff --git a/src/artifacts/inventory/ArtifactOccurrenceIdentity.test.ts b/src/artifacts/inventory/ArtifactOccurrenceIdentity.test.ts index 1d2ea42ae0..9d7fe276f6 100644 --- a/src/artifacts/inventory/ArtifactOccurrenceIdentity.test.ts +++ b/src/artifacts/inventory/ArtifactOccurrenceIdentity.test.ts @@ -9,7 +9,6 @@ import { compareArtifacts } from "../../domain/artifactComparison.js"; import { parseArtifactInventoryEvidence } from "../../domain/artifactInventoryEvidence.js"; import { projectAppleInventory } from "../../domain/apple/appleApplication.js"; import { createEvidence } from "../../domain/evidence.js"; -import { jsonValueSchema } from "../../domain/jsonValue.js"; import { thinMach } from "../../domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; @@ -32,7 +31,7 @@ const observeArchive = async (order: readonly string[]) => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(inventory), + result: inventory, confidence: "observed", authority: "shipped-artifact", }, @@ -128,7 +127,7 @@ describe("content identity and occurrence facts", () => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(inventory), + result: inventory, confidence: "observed", authority: "shipped-artifact", }, diff --git a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts index 898a0995fc..95bd2de054 100644 --- a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts @@ -19,7 +19,6 @@ import { import { readPyzArchive } from "./PyzArchive.js"; import { marshalCode } from "../../python/PythonPyc.fixture.js"; import { createEvidence } from "../../domain/evidence.js"; -import { jsonValueSchema } from "../../domain/jsonValue.js"; import { parseArtifactInventoryEvidence } from "../../domain/artifactInventoryEvidence.js"; import { ArtifactReaderFailure, @@ -365,7 +364,7 @@ it.each([true, false])( parameters: { page: index }, confidence: "observed", authority: "shipped-artifact", - result: jsonValueSchema.parse({ + result: { ...inventory, nodes: index === 0 ? inventory.nodes : [], edges: index === 0 ? inventory.edges : [], @@ -378,7 +377,7 @@ it.each([true, false])( row, ] : [row], - }), + }, }, ), ); diff --git a/src/browser/PlaywrightElectronActiveActions.ts b/src/browser/PlaywrightElectronActiveActions.ts index d06657e86b..3826e64824 100644 --- a/src/browser/PlaywrightElectronActiveActions.ts +++ b/src/browser/PlaywrightElectronActiveActions.ts @@ -373,7 +373,6 @@ const emitDeepLink = ( ); /** Read the current Electron windows, metrics, and hook snapshot. */ -// oxlint-disable-next-line max-lines-per-function -- one state read must share one capture boundary. export const readApplicationState = async ( application: ElectronApplication, ): Promise<{ diff --git a/src/domain/apple/appleApplication.test.ts b/src/domain/apple/appleApplication.test.ts index ccb00c333d..8a9685a887 100644 --- a/src/domain/apple/appleApplication.test.ts +++ b/src/domain/apple/appleApplication.test.ts @@ -10,7 +10,6 @@ import { AnalysisInputError } from "../analysisErrorCore.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { digestCanonicalValue } from "../canonicalDigest.js"; import { createEvidence, type Evidence } from "../evidence.js"; -import { jsonValueSchema } from "../jsonValue.js"; type EntryKind = "file" | "directory" | "symlink" | "slice"; type RootFormat = "ipa" | "zip" | "dmg" | "directory" | "asar"; @@ -136,7 +135,7 @@ const inventoryEvidence = ( parameters: {}, confidence: "observed", authority: "shipped-artifact", - result: jsonValueSchema.parse({ + result: { manifest: { manifest_id: `agm_${digestCanonicalValue( { root_artifact_id: rootId, graph_sha256: graphSha256 }, @@ -157,7 +156,7 @@ const inventoryEvidence = ( integrity_contradictions: [], coverage: { scope: "container-entries", status: "complete" }, limitations: [], - }), + }, }, ); }; diff --git a/src/domain/artifactInventoryEvidence.test.ts b/src/domain/artifactInventoryEvidence.test.ts index 9ab18f2d51..0a5508a876 100644 --- a/src/domain/artifactInventoryEvidence.test.ts +++ b/src/domain/artifactInventoryEvidence.test.ts @@ -14,7 +14,6 @@ import { parseArtifactInspectionInventoryResult, } from "./artifactInventoryEvidence.js"; import { createEvidence } from "./evidence.js"; -import { jsonValueSchema } from "./jsonValue.js"; const inventoryEvidence = ARTIFACT_COMPARISON_EXAMPLE.left; const target = inventoryEvidence.subject; @@ -31,7 +30,7 @@ const inspectionEvidence = (result: unknown, subject = target) => { operation: "inspect_artifact", parameters: {}, - result: jsonValueSchema.parse(result), + result, confidence: "observed", authority: "shipped-artifact", }, @@ -152,12 +151,12 @@ it("keeps a commitment failure discovered across individually incomplete pages a parameters: {}, confidence: "observed", authority: "shipped-artifact", - result: jsonValueSchema.parse({ + result: { ...complete, manifest, nodes: includeNodes ? complete.nodes : [], occurrences: includeNodes ? [] : complete.occurrences, - }), + }, }, ); const nodes = page(true); diff --git a/src/domain/changedBehavior.test.ts b/src/domain/changedBehavior.test.ts index 8686065a43..4142623361 100644 --- a/src/domain/changedBehavior.test.ts +++ b/src/domain/changedBehavior.test.ts @@ -7,7 +7,7 @@ import { } from "./changedBehavior.js"; import { createEvidence, type Evidence } from "./evidence.js"; import { functionComparisonResultSchema } from "./functionComparisonSchemas.js"; -import { jsonValueSchema, type JsonValue } from "./jsonValue.js"; +import type { JsonValue } from "./jsonValue.js"; const source = (digit: string): Evidence => createEvidence( @@ -350,13 +350,11 @@ describe("changed-behavior input and Evidence validation", () => { predicateType: base.predicate_type, operation: base.operation, parameters: base.parameters, - result: jsonValueSchema.parse( - functionComparisonResultSchema.parse({ - ...result, - dimensions, - changes: dimensions.filter(({ status }) => status !== "unchanged"), - }), - ), + result: functionComparisonResultSchema.parse({ + ...result, + dimensions, + changes: dimensions.filter(({ status }) => status !== "unchanged"), + }), confidence: base.confidence, authority: base.authority, evidenceLinks: base.evidence_links, diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index f61a6babd8..6f2f767cdb 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -664,7 +664,7 @@ describe("function comparison ordering controls", () => { parameters: enhancedInputSchemas.analyze_function.parse({ procedure: "dispatch", }), - result: jsonValueSchema.parse(result), + result, confidence: "derived", authority: "shipped-artifact", relationshipCoverage: { status: "complete" }, diff --git a/src/domain/staticRuntimeCorrelation.test.ts b/src/domain/staticRuntimeCorrelation.test.ts index 0081495239..1fe58af428 100644 --- a/src/domain/staticRuntimeCorrelation.test.ts +++ b/src/domain/staticRuntimeCorrelation.test.ts @@ -35,11 +35,9 @@ const artifactComparison = createEvidence( left_evidence_ids: [ARTIFACT_COMPARISON_EXAMPLE.left.evidence_id], right_evidence_ids: [ARTIFACT_COMPARISON_EXAMPLE.right.evidence_id], }, - result: jsonValueSchema.parse( - compareArtifacts( - ARTIFACT_COMPARISON_EXAMPLE.left, - ARTIFACT_COMPARISON_EXAMPLE.right, - ), + result: compareArtifacts( + ARTIFACT_COMPARISON_EXAMPLE.left, + ARTIFACT_COMPARISON_EXAMPLE.right, ), confidence: "derived", authority: "analyst-inference", @@ -157,10 +155,10 @@ it("keeps text, assembly and unavailable CFG caveats on their selected correlati { operation: source.operation, parameters: source.parameters, - result: jsonValueSchema.parse({ + result: { ...functionDossierSchema.parse(source.normalized_result), basic_blocks: [{ start: "0x1000", end: "0x1000", successors: [] }], - }), + }, confidence: source.confidence, authority: source.authority, }, @@ -178,7 +176,7 @@ it("keeps text, assembly and unavailable CFG caveats on their selected correlati left_evidence_id: left.evidence_id, right_evidence_id: right.evidence_id, }, - result: jsonValueSchema.parse(computed), + result: computed, confidence: "derived", authority: "analyst-inference", limitations: computed.limitations, diff --git a/src/domain/webMcpDiscovery.test.ts b/src/domain/webMcpDiscovery.test.ts index 452038d6b5..29c57973a0 100644 --- a/src/domain/webMcpDiscovery.test.ts +++ b/src/domain/webMcpDiscovery.test.ts @@ -6,7 +6,6 @@ import { parseEvidenceBundle, serializeEvidenceBundle, } from "./evidenceBundle.js"; -import { jsonValueSchema } from "./jsonValue.js"; import { webMcpDiscoverySchema, type WebMcpDiscovery, @@ -106,7 +105,7 @@ describe("WebMCP discovery consequential annotation", () => { predicateType: "rea.webmcp-discovery", operation: "discover_webmcp_tools", parameters: {}, - result: jsonValueSchema.parse(result), + result, }); const bundle = createEvidenceBundle([evidence]); const reread = parseEvidenceBundle( diff --git a/tests/acceptance/applications/artifactComparisonCoverage.test.ts b/tests/acceptance/applications/artifactComparisonCoverage.test.ts index 94c0ceab1b..b60d46da19 100644 --- a/tests/acceptance/applications/artifactComparisonCoverage.test.ts +++ b/tests/acceptance/applications/artifactComparisonCoverage.test.ts @@ -14,7 +14,7 @@ import { parseEvidence, type Evidence, } from "../../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; @@ -75,7 +75,7 @@ const assertCoverageControls = async ({ { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(result), + result: result, confidence: "derived", authority: "shipped-artifact", }, diff --git a/tests/boundary/browser/cdpCleanupCancellation.test.ts b/tests/boundary/browser/cdpCleanupCancellation.test.ts index 1c31341d48..8c59661d8d 100644 --- a/tests/boundary/browser/cdpCleanupCancellation.test.ts +++ b/tests/boundary/browser/cdpCleanupCancellation.test.ts @@ -1,4 +1,4 @@ -import { rm, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; @@ -145,7 +145,6 @@ it("closes an Electron inspection session when cleanup is cancelled", async () = }); onTestFinished(async () => { await browser.close(); - await rm(root, { recursive: true, force: true }); }); const controller = new AbortController(); const pending = new CdpElectronProvider().inspectPage( @@ -204,7 +203,6 @@ it("classifies cancellation that arrives during final Electron detach", async () }); onTestFinished(async () => { await browser.close(); - await rm(root, { recursive: true, force: true }); }); const result = await new CdpElectronProvider().inspectPage( @@ -283,7 +281,6 @@ it.each( closing.mockRestore(); await provider.close(); await browser.close(); - await rm(root, { recursive: true, force: true }); }); const result = provider instanceof CdpBrowserProvider diff --git a/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts b/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts index 9a3582bf5b..edb801456b 100644 --- a/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts +++ b/tests/boundary/browser/cdpProviderCleanupOwnership.test.ts @@ -1,4 +1,4 @@ -import { rm, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; @@ -52,7 +52,6 @@ it.each([ | undefined; if (closeProvider !== undefined) await closeProvider.call(provider); await browser.close(); - await rm(root, { recursive: true, force: true }); }); const close = vi diff --git a/tests/boundary/filesystem/artifactComparison.test.ts b/tests/boundary/filesystem/artifactComparison.test.ts index 4b9b07321f..e15fdbf4b6 100644 --- a/tests/boundary/filesystem/artifactComparison.test.ts +++ b/tests/boundary/filesystem/artifactComparison.test.ts @@ -36,7 +36,7 @@ const observe = async (path: string) => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(inventory), + result: inventory, confidence: "observed", authority: "shipped-artifact", }, @@ -62,7 +62,7 @@ const inspectPages = ( { operation: "inspect_artifact", parameters: {}, - result: jsonValueSchema.parse({ + result: { inspection_id: `ai_${"f".repeat(64)}`, subject: { manifest_id: inventory.manifest.manifest_id, @@ -96,7 +96,7 @@ const inspectPages = ( }, evidence_links: pages.map(({ evidence_id }) => evidence_id), limitations: [], - }), + }, confidence: "observed", authority: "shipped-artifact", }, @@ -144,7 +144,7 @@ describe("leaf artifact comparison", () => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(withoutCoverage), + result: withoutCoverage, confidence: "observed", authority: "shipped-artifact", }, @@ -306,11 +306,11 @@ describe("artifact inventory coverage", () => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...graph, coverage: { scope: "root", status: "unknown" }, limitations: ["No extra child entries were observed."], - }), + }, confidence: "derived", authority: "shipped-artifact", }, @@ -348,10 +348,10 @@ describe("artifact inventory coverage", () => { { operation: "inventory_artifact", parameters: fragment.parameters, - result: jsonValueSchema.parse({ + result: { ...fragmentGraph, coverage: { ...fragmentGraph.coverage, status: "unknown" }, - }), + }, confidence: "derived", authority: "shipped-artifact", }, @@ -448,10 +448,10 @@ describe("artifact comparison", () => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...completeInventory, nodes: [], - }), + }, confidence: "observed", authority: "shipped-artifact", limitations: ["Inventory node observations are unavailable."], @@ -496,10 +496,10 @@ describe("artifact comparison completeness", () => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...inventory, coverage: { scope: "root", status: "complete" }, - }), + }, confidence: "derived", authority: "shipped-artifact", }, @@ -862,10 +862,10 @@ describe("artifact comparison fragment integrity", () => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...artifactInventoryResultSchema.parse(fragment.normalized_result), coverage: { scope: "root", status: "complete" }, - }), + }, }, ); expect(() => diff --git a/tests/boundary/filesystem/artifactComparisonSummary.test.ts b/tests/boundary/filesystem/artifactComparisonSummary.test.ts index 5042c916c3..ef1e911a60 100644 --- a/tests/boundary/filesystem/artifactComparisonSummary.test.ts +++ b/tests/boundary/filesystem/artifactComparisonSummary.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from "vitest"; import { inventoryArtifact } from "../../fixtures/artifactInventory.js"; import { compareArtifacts } from "../../../src/domain/artifactComparison.js"; import { createEvidence } from "../../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const observe = async ( @@ -30,7 +30,7 @@ const observe = async ( { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(inventory), + result: inventory, confidence: partial ? "derived" : "observed", authority: "shipped-artifact", }, diff --git a/tests/boundary/filesystem/artifactDirectoryIdentity.test.ts b/tests/boundary/filesystem/artifactDirectoryIdentity.test.ts index 3d1d252100..258f1fe098 100644 --- a/tests/boundary/filesystem/artifactDirectoryIdentity.test.ts +++ b/tests/boundary/filesystem/artifactDirectoryIdentity.test.ts @@ -10,7 +10,7 @@ import { inventoryArtifact } from "../../fixtures/artifactInventory.js"; import { compareArtifacts } from "../../../src/domain/artifactComparison.js"; import type { ArtifactInventoryResult } from "../../../src/domain/artifactGraph.js"; import { createEvidence } from "../../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const observeZip = async (entries: readonly (readonly [string, string])[]) => { @@ -32,7 +32,7 @@ const observeZip = async (entries: readonly (readonly [string, string])[]) => { { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(inventory), + result: inventory, confidence: "observed", authority: "shipped-artifact", }, diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index ce3217f79b..87b472c1fe 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -1360,6 +1360,30 @@ it.each([ }, ); +it.each(["knip:production", "lint:typed"])( + "keeps the %s analysis command within static checks", + async (script) => { + const { directory } = await fixture(); + const base = await change( + directory, + ["package.json"], + JSON.stringify({ scripts: { [script]: "node scripts/old.mjs" } }), + ); + const head = await change( + directory, + ["package.json"], + JSON.stringify({ scripts: { [script]: "node scripts/new.mjs" } }), + ); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + runtime: false, + package: false, + evm: false, + browser: false, + }); + }, +); + it("retains a broad fallback when a provider script adds shared shell work", async () => { const { directory } = await fixture(); const before = { diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index 0c48a5ef06..3a442708bd 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -14,7 +14,7 @@ import { import { reconcileJavaScriptRuntimeEvidence } from "../../../src/application/javascript/JavaScriptRuntimeReconciliationService.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { applicationVersionComparisonResultSchema, @@ -377,10 +377,10 @@ describe("Application graph Evidence envelopes", () => { predicateType: source.predicate_type, operation: source.operation, parameters: source.parameters, - result: jsonValueSchema.parse({ + result: { ...result, evidence_links: [...result.evidence_links, extraLink], - }), + }, rawResult: source.raw_result, confidence: source.confidence, authority: source.authority, @@ -479,7 +479,7 @@ describe("JavaScript application workflow error diagnostics", () => { predicateType: evidence.predicate_type, operation: evidence.operation, parameters: evidence.parameters, - result: jsonValueSchema.parse(malformedResult), + result: malformedResult, confidence: evidence.confidence, authority: evidence.authority, limitations: evidence.limitations, diff --git a/tests/boundary/filesystem/lintRuleConfiguration.test.ts b/tests/boundary/filesystem/lintRuleConfiguration.test.ts new file mode 100644 index 0000000000..fb5dc363cf --- /dev/null +++ b/tests/boundary/filesystem/lintRuleConfiguration.test.ts @@ -0,0 +1,270 @@ +import { parse } from "@babel/parser"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { writeRepositoryOxlintConfiguration } from "../../fixtures/oxlintConfiguration.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { processTest } from "../../support/process/processFixture.js"; + +const repository = fileURLToPath(new URL("../../../", import.meta.url)); +const oxlint = join(repository, "node_modules/oxlint/bin/oxlint"); +const applicationSource = "src/application/lintRuleAcceptance.ts"; +const testSource = "tests/boundary/lintRuleAcceptance.test.ts"; + +interface AcceptanceCase { + readonly name: string; + readonly filename: string; + readonly source: string; + readonly exitCode: number; + readonly diagnostics: readonly { + readonly code?: string; + readonly severity: "error" | "warning"; + readonly message?: string; + }[]; +} + +const cases: readonly AcceptanceCase[] = [ + { + name: "enables imported Evidence JSON admission errors through the repository plugin", + filename: applicationSource, + source: ` + import { createEvidence as record } from "../domain/evidence.js"; + import { jsonValueSchema as json } from "../domain/jsonValue.js"; + export const evidence = record(undefined, {}, { result: json.parse({ ready: true }) }); + `, + exitCode: 1, + diagnostics: [ + { code: "rea(no-evidence-json-preadmission)", severity: "error" }, + ], + }, + { + name: "recognizes canonical Evidence owners imported through file URLs", + filename: applicationSource, + source: ` + import { createEvidence as record } from ${JSON.stringify(pathToFileURL(join(repository, "src/domain/evidence.js")).href)}; + import { jsonValueSchema as json } from ${JSON.stringify(pathToFileURL(join(repository, "src/domain/jsonValue.js")).href)}; + export const evidence = record(undefined, {}, { result: json.parse({ ready: true }) }); + `, + exitCode: 1, + diagnostics: [ + { code: "rea(no-evidence-json-preadmission)", severity: "error" }, + ], + }, + { + name: "allows semantic schema admission and stored JSON contracts", + filename: applicationSource, + source: ` + import { z } from "zod"; + import { createEvidence } from "../domain/evidence.js"; + import { jsonValueSchema } from "../domain/jsonValue.js"; + const semanticSchema = z.object({ ready: z.boolean() }); + export const semantic = createEvidence(undefined, {}, { result: semanticSchema.parse({ ready: true }) }); + const result = jsonValueSchema.parse({ ready: true }); + export const stored = createEvidence(undefined, {}, { result }); + `, + exitCode: 0, + diagnostics: [], + }, + { + name: "enables positional catalog type errors through the repository plugin", + filename: applicationSource, + source: ` + import { GPU_TOOL_CONTRACTS as contracts } from "../contracts/gpu/gpuToolContracts.js"; + export type Selected = (typeof contracts)[0]; + `, + exitCode: 1, + diagnostics: [ + { code: "rea(no-positional-tool-contract)", severity: "error" }, + ], + }, + { + name: "enables duplicate fixture root teardown errors through the repository plugin", + filename: testSource, + source: ` + import { afterEach } from "vitest"; + import { rm } from "node:fs/promises"; + import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + const root = await createTestTempDirectory("rea-lint-rule-"); + afterEach(() => rm(root, { recursive: true, force: true })); + `, + exitCode: 1, + diagnostics: [{ code: "rea(no-fixture-root-cleanup)", severity: "error" }], + }, + { + name: "allows explicit root deletion outside fixture teardown", + filename: testSource, + source: ` + import { rm } from "node:fs/promises"; + import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + const root = await createTestTempDirectory("rea-lint-rule-"); + await rm(root, { recursive: true, force: true }); + `, + exitCode: 0, + diagnostics: [], + }, + { + name: "rejects an unreturned promise callback containing assertions", + filename: testSource, + source: ` + import { expect, it } from "vitest"; + it("unjoined assertion", () => { + Promise.resolve("ready").then(value => { expect(value).toBe("ready"); }); + }); + `, + exitCode: 1, + diagnostics: [ + { code: "vitest(valid-expect-in-promise)", severity: "error" }, + ], + }, + { + name: "rejects focused tests through the repository Vitest configuration", + filename: testSource, + source: ` + import { expect, it } from "vitest"; + it.only("focused assertion", () => { expect("ready").toBe("ready"); }); + `, + exitCode: 1, + diagnostics: [{ code: "vitest(no-focused-tests)", severity: "error" }], + }, + { + name: "keeps native stored assertion limitations at warning severity", + filename: testSource, + source: ` + import { expect, it } from "vitest"; + it("joins a stored assertion", async () => { + const assertion = expect(Promise.reject(new Error("controlled"))).rejects.toThrow("controlled"); + await assertion; + }); + `, + exitCode: 0, + diagnostics: [{ code: "vitest(valid-expect)", severity: "warning" }], + }, + { + name: "allows an assertion diagnostic message", + filename: testSource, + source: ` + import { expect, it } from "vitest"; + it("keeps diagnostic context", () => { + expect("ready", "controlled diagnostic context").toBe("ready"); + }); + `, + exitCode: 0, + diagnostics: [], + }, + { + name: "rejects unused suppression directives through the repository configuration", + filename: testSource, + source: ` + // oxlint-disable-next-line no-var + export const ready = true; + `, + exitCode: 1, + diagnostics: [ + { + severity: "error", + message: "Unused oxlint-disable directive (no problems were reported).", + }, + ], + }, +]; + +const reportSchema = z.object({ + diagnostics: z.array( + z.object({ + code: z.string().optional(), + severity: z.enum(["error", "warning"]), + message: z.string(), + }), + ), +}); + +const relocateImports = (scenario: AcceptanceCase, path: string): string => { + const source = scenario.source; + const parsed = parse(source, { + sourceType: "module", + plugins: ["typescript"], + }); + const replacements = []; + for (const statement of parsed.program.body) { + if ( + statement.type !== "ImportDeclaration" || + !statement.source.value.startsWith(".") + ) + continue; + const target = resolve( + repository, + dirname(scenario.filename), + statement.source.value, + ); + const relativePath = relative(dirname(path), target); + const specifier = isAbsolute(relativePath) + ? pathToFileURL(target).href + : relativePath.startsWith(".") + ? relativePath.split(sep).join("/") + : `./${relativePath.split(sep).join("/")}`; + const { start, end } = statement.source; + if ( + start === null || + start === undefined || + end === null || + end === undefined + ) + throw new Error("Authored lint fixture import has no source range"); + replacements.push({ + start, + end, + specifier, + }); + } + let relocated = source; + for (const replacement of replacements.reverse()) + relocated = + relocated.slice(0, replacement.start) + + JSON.stringify(replacement.specifier) + + relocated.slice(replacement.end); + return relocated; +}; + +for (const scenario of cases) { + processTest(scenario.name, async ({ processes }) => { + const project = await createTestTempDirectory("rea-lint-configuration-"); + const configuration = await writeRepositoryOxlintConfiguration(project); + const path = join(project, scenario.filename); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, relocateImports(scenario, path)); + const result = await processes.run( + process.execPath, + [ + oxlint, + "--config", + configuration, + "--threads", + "1", + "--format", + "json", + scenario.filename, + ], + { + cwd: project, + timeoutMs: 15_000, + }, + ); + expect(result.signal, result.stderr).toBeNull(); + expect(result.exitCode, result.stderr || result.stdout).toBe( + scenario.exitCode, + ); + const { diagnostics } = reportSchema.parse(JSON.parse(result.stdout)); + expect( + diagnostics.map(({ code, severity }) => ({ code, severity })), + ).toEqual( + scenario.diagnostics.map(({ code, severity }) => ({ code, severity })), + ); + for (const [index, expected] of scenario.diagnostics.entries()) { + if (expected.message !== undefined) + expect(diagnostics[index]?.message).toBe(expected.message); + } + }); +} diff --git a/tests/boundary/filesystem/productionReachabilityReport.test.ts b/tests/boundary/filesystem/productionReachabilityReport.test.ts new file mode 100644 index 0000000000..b7c6c6a71d --- /dev/null +++ b/tests/boundary/filesystem/productionReachabilityReport.test.ts @@ -0,0 +1,117 @@ +import { spawnSync } from "node:child_process"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it } from "vitest"; + +import { parseProductionReachabilityReport } from "../../../scripts/ci/production-reachability-report.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const script = fileURLToPath( + new URL( + "../../../scripts/ci/production-reachability-report.mjs", + import.meta.url, + ), +); +const configuration = new URL("../../../knip.production.json", import.meta.url); +const reportPath = (root: string) => + join(root, ".cache/static-checks/production-reachability.json"); +const invoke = (root: string) => + spawnSync(process.execPath, [script], { + cwd: root, + encoding: "utf8", + timeout: 15_000, + }); + +it("reports test-only candidates without losing dynamic, worker or published roots", async () => { + const root = await createTestTempDirectory("rea-production-reachability-"); + const files = { + "package.json": JSON.stringify({ + name: "reachability-fixture", + type: "module", + bin: { fixture: "scripts/rea.mjs" }, + exports: { "./published": "./src/published.ts" }, + }), + "src/main.ts": + 'import { live } from "./facts.js"; console.log(live); void import("./lazy.js");', + "src/facts.ts": "export const live = 1; export const testOnly = 2;", + "src/lazy.ts": "console.log('dynamic production module');", + "src/testOnly.ts": "export const retired = 3;", + "src/published.ts": "export const publicContract = 4;", + "src/composition/javascriptAnalysisWorkerProcess.ts": + 'import { workerFact } from "../workerFacts.js"; console.log(workerFact);', + "src/workerFacts.ts": "export const workerFact = 5;", + "scripts/rea.mjs": 'import "../src/main.js";', + "tests/facts.test.ts": + 'import { testOnly } from "../src/facts.js"; import { retired } from "../src/testOnly.js"; console.log(testOnly, retired);', + }; + for (const [path, content] of Object.entries(files)) { + const destination = join(root, path); + await mkdir(dirname(destination), { recursive: true }); + await writeFile(destination, content); + } + await writeFile( + join(root, "knip.production.json"), + await readFile(configuration, "utf8"), + ); + + const result = invoke(root); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("advisory candidates"); + const { issues } = parseProductionReachabilityReport( + await readFile(reportPath(root), "utf8"), + ); + expect(issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + file: "src/facts.ts", + exports: expect.arrayContaining([ + expect.objectContaining({ name: "testOnly" }), + ]), + }), + expect.objectContaining({ file: "src/testOnly.ts" }), + ]), + ); + const reached = [ + "src/main.ts", + "src/lazy.ts", + "src/published.ts", + "src/workerFacts.ts", + "src/composition/javascriptAnalysisWorkerProcess.ts", + "tests/facts.test.ts", + ]; + expect(issues.filter(({ file }) => reached.includes(file))).toEqual([]); + expect( + issues.flatMap(({ exports: findings = [] }) => + findings.map(({ name }) => name), + ), + ).not.toContain("live"); +}); + +it("fails on Knip configuration errors and removes a stale candidate report", async () => { + const root = await createTestTempDirectory( + "rea-production-reachability-error-", + ); + await writeFile(join(root, "package.json"), '{"name":"broken-config"}'); + await writeFile(join(root, "knip.production.json"), "{"); + await mkdir(dirname(reportPath(root)), { recursive: true }); + await writeFile(reportPath(root), '{"issues":[]}'); + + const result = invoke(root); + expect(result.error).toBeUndefined(); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Production reachability analysis failed"); + await expect(access(reportPath(root))).rejects.toMatchObject({ + code: "ENOENT", + }); +}); + +it.each([ + "not JSON", + '{"issues":[{"file":"src/facts.ts","exports":[{"name":42}]}]}', + '{"issues":[{"file":"src/facts.ts","unrecognized":[]}]}', +])("rejects malformed checker output: %s", (text) => { + expect(() => parseProductionReachabilityReport(text)).toThrow(); +}); diff --git a/tests/boundary/filesystem/staticCheckReports.test.ts b/tests/boundary/filesystem/staticCheckReports.test.ts index c133e21cfd..b433dc9f55 100644 --- a/tests/boundary/filesystem/staticCheckReports.test.ts +++ b/tests/boundary/filesystem/staticCheckReports.test.ts @@ -1,10 +1,11 @@ import { spawnSync } from "node:child_process"; -import { access, cp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; import { z } from "zod"; +import { writeRepositoryOxlintConfiguration } from "../../fixtures/oxlintConfiguration.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const script = fileURLToPath( @@ -149,11 +150,11 @@ it("keeps lint errors failing and visible alongside the retained warnings", asyn it("excludes translation staging while retaining authored website errors", async () => { const root = await fixture(); - for (const name of [".gitignore", ".oxlintrc.json"]) - await writeFile(join(root, name), await readFile(join(repository, name))); - await cp(join(repository, "scripts/lint"), join(root, "scripts/lint"), { - recursive: true, - }); + await writeFile( + join(root, ".gitignore"), + await readFile(join(repository, ".gitignore")), + ); + await writeRepositoryOxlintConfiguration(root); const authored = "website/public/assets/authored.js"; const staged = "website/.i18n-regression/site/examples/notes-electron/main.js"; @@ -166,7 +167,7 @@ it("excludes translation staging while retaining authored website errors", async ); } - const result = invoke(root, "lint", "website"); + const result = invoke(root, "lint"); expect(result.error).toBeUndefined(); expect(result.status).toBe(1); expect(result.stdout).toContain("1 errors, 0 warnings"); diff --git a/tests/boundary/filesystem/typedLintRules.test.ts b/tests/boundary/filesystem/typedLintRules.test.ts new file mode 100644 index 0000000000..74694abb22 --- /dev/null +++ b/tests/boundary/filesystem/typedLintRules.test.ts @@ -0,0 +1,192 @@ +import { mkdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { writeRepositoryOxlintConfiguration } from "../../fixtures/oxlintConfiguration.js"; +import { processTest } from "../../support/process/processFixture.js"; + +const dependencies = fileURLToPath( + new URL("../../../node_modules", import.meta.url), +); +const reporter = fileURLToPath( + new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url), +); +const sources = { + "src/unjoined.test.ts": ` +import { expect, it } from "vitest"; + +const cliTest = it.extend<{ label: string }>({ label: "owned" }); + +it("forgets the assertion promise", () => { + expect(Promise.resolve("ready")).resolves.toBe("ready"); +}); + +cliTest("forgets the assertion in an extended callback", ({ label }) => { + expect(Promise.resolve(label)).resolves.toBe("owned"); +}); +`, + "src/joined.test.ts": ` +import { expect, it } from "vitest"; + +const cliTest = it.extend<{ label: string }>({ label: "owned" }); + +it("joins a stored assertion promise", async () => { + const assertion = expect(Promise.resolve("ready")).resolves.toBe("ready"); + await assertion; +}); + +cliTest("joins a stored assertion in an extended callback", async ({ label }) => { + const assertion = expect(Promise.resolve(label)).resolves.toBe("owned"); + await assertion; +}); + +export async function finishBeforeCleanup(): Promise { + try { + return await Promise.resolve("ready"); + } finally { + await Promise.resolve(); + } +} +`, + "src/misused.ts": ` +export function promiseCondition(): string { + if (Promise.resolve(false)) return "ready"; + return "pending"; +} +`, + "src/nonexhaustive.ts": ` +type Outcome = { kind: "open" } | { kind: "closed" }; + +export function describeOutcome(outcome: Outcome): string { + switch (outcome.kind) { + case "open": return "open"; + default: return "unknown"; + } +} +`, + "src/unawaited-finally.ts": ` +export async function cleanupBeforeCompletion(): Promise { + try { + return Promise.resolve("ready"); + } finally { + await Promise.resolve(); + } +} +`, + "src/test-misused.test.ts": ` +import { it } from "vitest"; +it("keeps test-only promise conditions advisory", () => { + if (Promise.resolve(false)) throw new Error("test fixture"); +}); +`, + "tests/unjoined.test.ts": ` +import { expect, it } from "vitest"; +const cliTest = it.extend<{ label: string }>({ label: "owned" }); +it("warns about an unjoined boundary-test assertion", () => { + expect(Promise.resolve("ready")).resolves.toBe("ready"); +}); +cliTest("warns in an extended boundary-test callback", ({ label }) => { + expect(Promise.resolve(label)).resolves.toBe("owned"); +}); +`, +}; + +processTest( + "runs typed promise and exhaustiveness rules on an isolated real TypeScript project", + async ({ processes }) => { + const root = await createTestTempDirectory("rea-typed-lint-"); + await symlink( + dependencies, + join(root, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + await Promise.all( + Object.keys(sources).map((name) => + mkdir(dirname(join(root, name)), { recursive: true }), + ), + ); + await writeRepositoryOxlintConfiguration(root); + await Promise.all([ + writeFile(join(root, "package.json"), '{"type":"module"}\n'), + writeFile( + join(root, "tsconfig.json"), + JSON.stringify({ + compilerOptions: { + strict: true, + noEmit: true, + skipLibCheck: true, + module: "NodeNext", + moduleResolution: "NodeNext", + target: "ES2023", + types: ["node"], + }, + files: Object.keys(sources), + }), + ), + ...Object.entries(sources).map(([name, source]) => + writeFile(join(root, name), source), + ), + ]); + + const result = await processes.run( + process.execPath, + [reporter, "typed-lint", ...Object.keys(sources)], + { + cwd: root, + timeoutMs: 20_000, + env: { ...process.env, GOMAXPROCS: "2", GOMEMLIMIT: "1GiB" }, + }, + ); + expect(result.signal, result.stderr).toBeNull(); + expect(result.exitCode, result.stderr || result.stdout).toBe(1); + const { diagnostics } = z + .object({ + diagnostics: z.array( + z.object({ + code: z.string(), + severity: z.enum(["error", "warning"]), + filename: z.string(), + }), + ), + }) + .parse( + JSON.parse( + await readFile( + join(root, ".cache/static-checks/typed-oxlint.json"), + "utf8", + ), + ), + ); + expect( + diagnostics + .filter(({ code }) => + [ + "typescript(no-floating-promises)", + "typescript(no-misused-promises)", + "typescript(return-await)", + "typescript(switch-exhaustiveness-check)", + ].includes(code), + ) + .map( + ({ code, filename, severity }) => + `${(isAbsolute(filename) ? relative(root, filename) : filename).split(sep).join("/")}: ${severity}: ${code}`, + ) + .sort(), + ).toEqual([ + "src/misused.ts: error: typescript(no-misused-promises)", + "src/nonexhaustive.ts: warning: typescript(switch-exhaustiveness-check)", + "src/test-misused.test.ts: warning: typescript(no-misused-promises)", + "src/unawaited-finally.ts: warning: typescript(return-await)", + "src/unjoined.test.ts: error: typescript(no-floating-promises)", + "src/unjoined.test.ts: error: typescript(no-floating-promises)", + "tests/unjoined.test.ts: warning: typescript(no-floating-promises)", + "tests/unjoined.test.ts: warning: typescript(no-floating-promises)", + ]); + expect( + diagnostics.filter(({ severity }) => severity === "error"), + ).toHaveLength(3); + }, +); diff --git a/tests/boundary/filesystem/webArtifactReadOwnership.test.ts b/tests/boundary/filesystem/webArtifactReadOwnership.test.ts index e378d7a38c..1caf6ee3d1 100644 --- a/tests/boundary/filesystem/webArtifactReadOwnership.test.ts +++ b/tests/boundary/filesystem/webArtifactReadOwnership.test.ts @@ -1,4 +1,4 @@ -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it, onTestFinished, vi } from "vitest"; @@ -43,12 +43,11 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; it("retains cancelled source read ownership until service close retries it", async () => { const root = await createTestTempDirectory("rea-web-artifact-read-owner-"); - onTestFinished(async () => { + onTestFinished(() => { injected.sourcePath = undefined; injected.abortController = undefined; injected.failClose = false; injected.closeAttempts = 0; - await rm(root, { recursive: true, force: true }); }); const source = 'import "./dep.js";'; const fixture = webModuleArtifactsFixture(source); diff --git a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts index 33be84d85e..842c086e18 100644 --- a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts +++ b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts @@ -20,7 +20,7 @@ import { } from "../../../src/domain/evidence.js"; import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { analysisViewResultSchema, type AnalysisViewResult, @@ -446,10 +446,10 @@ it.each(["full", "compact"] as const)( { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...graph, occurrences: graph.occurrences.slice(0, 1), - }), + }, }, ); const partialReply = await client.callTool({ diff --git a/tests/boundary/mcp/artifactMcp.test.ts b/tests/boundary/mcp/artifactMcp.test.ts index 2227cec5df..4930077f4d 100644 --- a/tests/boundary/mcp/artifactMcp.test.ts +++ b/tests/boundary/mcp/artifactMcp.test.ts @@ -12,7 +12,7 @@ import { } from "../../../src/domain/evidence.js"; import { EvidenceIntegrityError } from "../../../src/domain/evidenceErrors.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; import { artifactEdgeId, @@ -468,7 +468,7 @@ it("records an explicitly continued mismatch, preserves verified siblings, and n { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(inventoryWithoutCoverage), + result: inventoryWithoutCoverage, }, ); const retainedComparison = await client.callTool({ @@ -913,7 +913,7 @@ const withForeignMetadataAndNestedZipMember = ( parameters: evidence.parameters, confidence: evidence.confidence, authority: evidence.authority, - result: jsonValueSchema.parse({ + result: { ...inventory, occurrences, edges, @@ -924,7 +924,7 @@ const withForeignMetadataAndNestedZipMember = ( graph_sha256: graphSha256, manifest_id: manifestId, }, - }), + }, }, ); }; diff --git a/tests/boundary/mcp/evidenceBundleInline.test.ts b/tests/boundary/mcp/evidenceBundleInline.test.ts index a27d2af1b6..060a1687a5 100644 --- a/tests/boundary/mcp/evidenceBundleInline.test.ts +++ b/tests/boundary/mcp/evidenceBundleInline.test.ts @@ -12,7 +12,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import { ANALYSIS_SNAPSHOT_TARGET } from "../../../src/domain/analysisSnapshot.fixture.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; import { createServer } from "../../../src/server/createServer.js"; @@ -195,7 +195,7 @@ const createBundleSourceSession = () => { predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, - result: jsonValueSchema.parse(EMPTY_PROCESS_CAPTURE_EXAMPLE), + result: EMPTY_PROCESS_CAPTURE_EXAMPLE, confidence: "observed", authority: "controlled-replay", }), diff --git a/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts index 8615d92d7f..fb9dbbd409 100644 --- a/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts @@ -14,7 +14,7 @@ import { type FunctionComparisonResult, } from "../../../src/domain/functionComparisonSchemas.js"; import { summarize } from "../../../src/domain/functionComparisonResults.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; @@ -165,7 +165,7 @@ it.each([ predicateType: base.predicate_type, operation: base.operation, parameters: base.parameters, - result: jsonValueSchema.parse(scenario.result(result)), + result: scenario.result(result), confidence: base.confidence, authority: base.authority, limitations: base.limitations, diff --git a/tests/boundary/mcp/functionComparisonMcp.test.ts b/tests/boundary/mcp/functionComparisonMcp.test.ts index 75bf38e384..c9380f8b97 100644 --- a/tests/boundary/mcp/functionComparisonMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonMcp.test.ts @@ -9,7 +9,6 @@ import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarnes import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; import { compareFunctions } from "../../../src/domain/functionComparison.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; @@ -74,7 +73,7 @@ it.each([ { operation: "analyze_function", parameters: { procedure: entry }, - result: jsonValueSchema.parse(value), + result: value, confidence: "derived", relationshipCoverage: { status: "complete" }, }, @@ -532,14 +531,14 @@ it.each([ }; const consistent = createEvidence(undefined, provider, { ...observation, - result: jsonValueSchema.parse(result), + result: result, }); const inconsistent = status === "unchanged" ? null : createEvidence(undefined, provider, { ...observation, - result: jsonValueSchema.parse({ ...result, status: "unchanged" }), + result: { ...result, status: "unchanged" }, }); const directory = await createTestTempDirectory("rea-function-status-"); const path = join(directory, "bundle.json"); diff --git a/tests/boundary/mcp/functionComparisonReferenceFactsMcp.test.ts b/tests/boundary/mcp/functionComparisonReferenceFactsMcp.test.ts index 842517d2fd..4e38faf6f5 100644 --- a/tests/boundary/mcp/functionComparisonReferenceFactsMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonReferenceFactsMcp.test.ts @@ -11,7 +11,7 @@ import { import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { functionComparisonResultSchema } from "../../../src/domain/functionComparisonSchemas.js"; import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { reconstructionVerificationResultSchema } from "../../../src/domain/reconstructionVerificationSchemas.js"; import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -147,7 +147,7 @@ const evidence = ( ? {} : { analysisProfile: options.profile ?? profile }), parameters: { ...example.parameters, observation: id }, - result: jsonValueSchema.parse({ + result: { ...dossier, procedure: { ...dossier.procedure, @@ -162,7 +162,7 @@ const evidence = ( callers: [...(options.callers ?? [])], incoming_references: [...(options.incoming ?? [])], outgoing_references: [...outgoing], - }), + }, confidence: "derived", authority: "shipped-artifact", relationshipCoverage: { status: "complete" }, diff --git a/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts b/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts index 6957f417c4..9f2aba5bef 100644 --- a/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts @@ -8,7 +8,7 @@ import { functionComparisonResultSchema, type FunctionComparisonResult as Comparison, } from "../../../src/domain/functionComparisonSchemas.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -30,7 +30,7 @@ const evidence = ( { operation: "analyze_function", parameters: { ...template.parameters, observation: id }, - result: jsonValueSchema.parse({ + result: { ...base, referenced_strings: [...strings], swift_string_literals: @@ -47,7 +47,7 @@ const evidence = ( }, ], swift_string_literal_omissions: [], - }), + }, confidence: template.confidence, authority: template.authority, relationshipCoverage: { status: "complete" }, diff --git a/tests/boundary/mcp/investigationAnalysisMcp.test.ts b/tests/boundary/mcp/investigationAnalysisMcp.test.ts index 5040c665af..25ca5d93da 100644 --- a/tests/boundary/mcp/investigationAnalysisMcp.test.ts +++ b/tests/boundary/mcp/investigationAnalysisMcp.test.ts @@ -1,4 +1,4 @@ -import { parseEvidence } from "../../../src/domain/evidence.js"; +import { parseEvidence, createEvidence } from "../../../src/domain/evidence.js"; import { ok as resultOk } from "../../../src/domain/result.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it } from "vitest"; @@ -14,18 +14,14 @@ import { } from "../../../src/application/AnalysisProvider.js"; import type { BinarySession } from "../../../src/application/binary/BinarySession.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; -import { INVESTIGATION_EXAMPLES } from "../../../src/contracts/investigationExamples.js"; import { + INVESTIGATION_EXAMPLES, FUNCTION_COMPARISON_EVIDENCE, PROCESS_CAPTURE_RECONSTRUCTION, PROCESS_CAPTURE_REFERENCE, PROCESS_COMPARISON_EVIDENCE, } from "../../../src/contracts/investigationExamples.js"; -import { createEvidence } from "../../../src/domain/evidence.js"; -import { - jsonObjectSchema, - jsonValueSchema, -} from "../../../src/domain/jsonValue.js"; +import { jsonObjectSchema } from "../../../src/domain/jsonValue.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; import type { RelationshipCoverage } from "../../../src/domain/relationshipCoverage.js"; @@ -217,7 +213,7 @@ const dossierWithCallees = (addresses: readonly string[]) => { predicateType: base.predicate_type, operation: base.operation, parameters: base.parameters, - result: jsonValueSchema.parse({ + result: { ...jsonObjectSchema.parse(base.normalized_result), callees: addresses.map((address) => ({ address, @@ -228,7 +224,7 @@ const dossierWithCallees = (addresses: readonly string[]) => { reason: "The fixture has no callee body-extent observation.", }, })), - }), + }, rawResult: base.raw_result, confidence: base.confidence, authority: base.authority, diff --git a/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts b/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts index 152f6379d6..6bf579b9c8 100644 --- a/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts +++ b/tests/boundary/mcp/managedWorkflowUnknownIdentityMcp.test.ts @@ -5,7 +5,7 @@ import { MANAGED_RECONSTRUCTION_IMPORT_EXAMPLE, } from "../../../src/contracts/managed/managedWorkflowExamples.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { managedMemberInspectionSchema, managedNativeBoundaryInspectionSchema, @@ -34,7 +34,7 @@ const emptyExportEvidence = () => { { operation: "inspect_macho", parameters: original.parameters, - result: jsonValueSchema.parse({ + result: { ...result, exports: { items: [], @@ -42,7 +42,7 @@ const emptyExportEvidence = () => { exhaustive: true, limitations: [], }, - }), + }, rawResult: null, limitations: [], }, @@ -63,7 +63,7 @@ const secondDeclaration = () => { return createEvidence(undefined, original.provider, { operation: "inspect_managed_native_boundaries", parameters: original.parameters, - result: jsonValueSchema.parse({ + result: { ...result, pinvoke_imports: [ { @@ -74,7 +74,7 @@ const secondDeclaration = () => { import_name: "open_native_again", }, ], - }), + }, rawResult: null, limitations: original.limitations, }); @@ -128,7 +128,7 @@ const memberEvidence = ( { operation: "inspect_managed_members", parameters: original.parameters, - result: jsonValueSchema.parse(normalized), + result: normalized, rawResult: null, limitations: normalized.limitations, }, diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts index a404dad098..b4fe9c8fdf 100644 --- a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts +++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts @@ -27,7 +27,7 @@ import { parseEvidence, type EvidenceProvider, } from "../../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { digestProcessCommitment } from "../../../src/domain/process/processScenario.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -45,14 +45,14 @@ const captureEvidence = ( predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: { side }, - result: jsonValueSchema.parse({ + result: { ...observation, manifest: { ...observation.manifest, comparison_contract: comparisonContract, comparison_contract_sha256: digestProcessCommitment(comparisonContract), }, - }), + }, confidence: "observed", authority: "controlled-replay", }); @@ -523,7 +523,7 @@ it("names the side and constraint when Evidence is not a usable capture", async predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, - result: jsonValueSchema.parse(result), + result: result, }); expect(parseEvidence(invalidResult)).toEqual(invalidResult); const malformed = await client.callTool({ @@ -565,7 +565,7 @@ it("keeps structural capture issues alongside semantic ones through MCP and the predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...rest, manifest, exit: { @@ -577,7 +577,7 @@ it("keeps structural capture issues alongside semantic ones through MCP and the ], }, }, - }), + }, }); const described = "(at manifest.completed_at: Invalid input: expected string, received undefined (1 more issues))"; diff --git a/tests/boundary/mcp/processComparisonVerificationMcp.test.ts b/tests/boundary/mcp/processComparisonVerificationMcp.test.ts index e906811d76..bac31269d7 100644 --- a/tests/boundary/mcp/processComparisonVerificationMcp.test.ts +++ b/tests/boundary/mcp/processComparisonVerificationMcp.test.ts @@ -5,7 +5,7 @@ import { expect, it, onTestFinished } from "vitest"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; import { compareProcessCaptures } from "../../../src/domain/process/processComparison.js"; @@ -48,11 +48,11 @@ it("rejects an inconsistent imported process comparison while retaining its Evid left_evidence_id: left.evidence_id, right_evidence_id: right.evidence_id, }, - result: jsonValueSchema.parse({ + result: { ...nativeResult, [dimension]: "contradiction", status, - }), + }, confidence: "derived", authority: "analyst-inference", evidenceLinks: [left.evidence_id, right.evidence_id], diff --git a/tests/boundary/mcp/processTraceAdaptersParity.test.ts b/tests/boundary/mcp/processTraceAdaptersParity.test.ts index 0fa2f948d3..e781ee9c30 100644 --- a/tests/boundary/mcp/processTraceAdaptersParity.test.ts +++ b/tests/boundary/mcp/processTraceAdaptersParity.test.ts @@ -11,7 +11,7 @@ import { compareProcessEvidenceFiles } from "../../../src/application/process/Pr import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; import { processTraceCaptureFixture } from "../../../src/domain/process/processCapture.fixture.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { createServer } from "../../../src/server/createServer.js"; import { observed as ok } from "../../fixtures/analysisExecution.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -25,7 +25,7 @@ const captureEvidence = (side: "left" | "right") => predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: { side }, - result: jsonValueSchema.parse(capture), + result: capture, confidence: "observed", authority: "controlled-replay", locations: [{ kind: "artifact-path", path: `/fixture/${side}` }], diff --git a/tests/boundary/mcp/promptCompletion.test.ts b/tests/boundary/mcp/promptCompletion.test.ts index 40dd6a370f..9bd76f71dd 100644 --- a/tests/boundary/mcp/promptCompletion.test.ts +++ b/tests/boundary/mcp/promptCompletion.test.ts @@ -184,7 +184,7 @@ describe("guided prompt completion from investigation records", () => { const inspectionEvidence = createEvidence(undefined, fixtureProvider, { operation: "inspect_artifact", parameters: {}, - result: jsonValueSchema.parse(inventoryInspection), + result: inventoryInspection, confidence: "observed", authority: "shipped-artifact", evidenceLinks: inventoryInspection.evidence_links, @@ -426,7 +426,7 @@ const createPagedInspectionEvidence = async ( { operation: "inventory_artifact", parameters: {}, - result: jsonValueSchema.parse(inventoryResult), + result: inventoryResult, confidence: "observed", authority: "shipped-artifact", }, @@ -530,7 +530,7 @@ const createPagedInspectionEvidence = async ( { operation: "inspect_artifact", parameters: {}, - result: jsonValueSchema.parse(result), + result: result, confidence: "observed", authority: "shipped-artifact", evidenceLinks: result.evidence_links, diff --git a/tests/boundary/mcp/serverUnknownsMcp.test.ts b/tests/boundary/mcp/serverUnknownsMcp.test.ts index f11d19b2d4..7c9b157db7 100644 --- a/tests/boundary/mcp/serverUnknownsMcp.test.ts +++ b/tests/boundary/mcp/serverUnknownsMcp.test.ts @@ -18,7 +18,7 @@ import { createServer } from "../../../src/server/createServer.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { parseEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; import { processCaptureSchema } from "../../../src/domain/process/processCapture.js"; @@ -153,7 +153,7 @@ it("records capture disagreement as a contradicted unknown", async () => { predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, - result: jsonValueSchema.parse(capture), + result: capture, confidence: "observed", authority: "controlled-replay", }); diff --git a/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts b/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts index 92779dd525..f1653c4ff3 100644 --- a/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts +++ b/tests/boundary/mcp/workflowUnknownIdentityMcp.test.ts @@ -4,7 +4,7 @@ import { expect, it } from "vitest"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; -import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; + import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; @@ -44,7 +44,7 @@ const pathSource = () => { ...template.parameters, observation: "workflow-unknown-path", }, - result: jsonValueSchema.parse({ + result: { ...dossier, callees: [ { @@ -54,7 +54,7 @@ const pathSource = () => { body: { available: false, reason: "No body was observed." }, }, ], - }), + }, confidence: template.confidence, authority: template.authority, relationshipCoverage: { status: "complete" }, @@ -76,10 +76,8 @@ const incompleteRight = (suffix: string) => { { operation: "analyze_function", parameters: { ...template.parameters, observation: suffix }, - result: jsonValueSchema.parse( - functionDossierSchema.parse( - FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, - ), + result: functionDossierSchema.parse( + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, ), confidence: template.confidence, authority: template.authority, diff --git a/tests/boundary/providers/managed/managedMemberComparison.test.ts b/tests/boundary/providers/managed/managedMemberComparison.test.ts index 2ddaa6018c..b579b83fe6 100644 --- a/tests/boundary/providers/managed/managedMemberComparison.test.ts +++ b/tests/boundary/providers/managed/managedMemberComparison.test.ts @@ -12,15 +12,17 @@ import { compareManagedMemberPaths, } from "../../../../src/application/managed/ManagedMemberComparisonService.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; -import { managedMemberComparisonResultSchema } from "../../../../src/domain/managed/managedMemberComparison.js"; +import { + managedMemberComparisonResultSchema, + compareManagedMembersInputSchema, +} from "../../../../src/domain/managed/managedMemberComparison.js"; import { readManagedMemberEvidence } from "../../../../src/domain/managed/managedMemberComparisonMatch.js"; import { createEvidence } from "../../../../src/domain/evidence.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; -import { jsonValueSchema } from "../../../../src/domain/jsonValue.js"; -import { compareManagedMembersInputSchema } from "../../../../src/domain/managed/managedMemberComparison.js"; + import { inspectManagedMembersBytes } from "../../../../src/dotnet/ManagedMemberInspector.js"; -import { managedPeFixtureTarget } from "../../../../src/dotnet/ManagedPe.fixture.js"; import { + managedPeFixtureTarget, buildManagedNestedTypeFixture, buildManagedPeFixture, } from "../../../../src/dotnet/ManagedPe.fixture.js"; @@ -139,7 +141,7 @@ const compare = ( { operation: "inspect_managed_members", parameters: {}, - result: jsonValueSchema.parse(change?.(inspection) ?? inspection), + result: change?.(inspection) ?? inspection, confidence: "observed", authority: "shipped-artifact", }, @@ -475,14 +477,14 @@ describe("managed member comparison subject identity", () => { { operation: "inspect_managed_members", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...inspection, artifact: { ...inspection.artifact, sha256: otherDigest }, identity_scope: { ...inspection.identity_scope, requires_artifact_sha256: otherDigest, }, - }), + }, }, ); const aligned = createEvidence( @@ -563,11 +565,11 @@ describe("managed call-target comparison", () => { { operation: "inspect_managed_members", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...inspection, call_edges: alterEdges?.(inspection.call_edges) ?? inspection.call_edges, - }), + }, confidence: "observed", authority: "shipped-artifact", }, @@ -636,10 +638,10 @@ describe("managed member comparison path workflow", () => { { operation: "inspect_managed_members", parameters: {}, - result: jsonValueSchema.parse({ + result: { ...inspection, coverage: { ...inspection.coverage, state: "partial" }, - }), + }, confidence: "observed", authority: "shipped-artifact", }, diff --git a/tests/evaluation/evidenceJsonPreadmissionRule.test.ts b/tests/evaluation/evidenceJsonPreadmissionRule.test.ts new file mode 100644 index 0000000000..fc435b3713 --- /dev/null +++ b/tests/evaluation/evidenceJsonPreadmissionRule.test.ts @@ -0,0 +1,265 @@ +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { RuleTester } from "oxlint/plugins-dev"; +import { describe, it } from "vitest"; + +import rule from "../../scripts/lint/rules/no-evidence-json-preadmission.mjs"; + +const filename = resolve("src/application/RuleFixture.ts"); +const imports = ` +import { createEvidence, createImmutableEvidenceSteps } from "../domain/evidence.js"; +import { jsonValueSchema, jsonObjectSchema } from "../domain/jsonValue.js"; +`; +const evidenceSource = new URL("../../src/domain/evidence.js", import.meta.url); +const jsonSource = new URL("../../src/domain/jsonValue.js", import.meta.url); +const fileImports = ` +import { createEvidence } from ${JSON.stringify(evidenceSource.href)}; +import { jsonValueSchema } from ${JSON.stringify(jsonSource.href)}; +`; +const absoluteImports = ` +import { createEvidence } from ${JSON.stringify(fileURLToPath(evidenceSource))}; +import { jsonValueSchema } from ${JSON.stringify(fileURLToPath(jsonSource))}; +`; +const valid = (name: string, code: string) => ({ name, filename, code }); +const invalid = (name: string, code: string, field: string) => ({ + name, + filename, + code, + errors: [{ messageId: "redundantJsonAdmission", data: { field } }], + output: null, +}); + +RuleTester.describe = describe; +RuleTester.it = it; +new RuleTester({ + languageOptions: { sourceType: "module", parserOptions: { lang: "ts" } }, +}).run("no-evidence-json-preadmission", rule, { + valid: [ + valid( + "file URL binding still respects lexical shadows", + `${fileImports} + function build(createEvidence) { createEvidence(target, provider, {result: jsonValueSchema.parse(input)}); }`, + ), + valid( + "HTTP modules are not filesystem owners", + ` + import { createEvidence } from "https://example.test/src/domain/evidence.js"; + import { jsonValueSchema } from "https://example.test/src/domain/jsonValue.js"; + createEvidence(target, provider, {result: jsonValueSchema.parse(input)});`, + ), + valid( + "malformed file URLs do not crash owner lookup", + ` + import {createEvidence} from "file://%"; + import {jsonValueSchema} from ${JSON.stringify(jsonSource.href)}; + createEvidence(target, provider, {result: jsonValueSchema.parse(input)});`, + ), + valid( + "queried file URL does not establish the canonical owner", + ` + import {createEvidence} from ${JSON.stringify(`${evidenceSource.href}?instance=other`)}; + import {jsonValueSchema} from ${JSON.stringify(jsonSource.href)}; + createEvidence(target, provider, {result: jsonValueSchema.parse(input)});`, + ), + valid( + "fragment file URL does not establish the canonical schema", + ` + import {createEvidence} from ${JSON.stringify(evidenceSource.href)}; + import {jsonValueSchema} from ${JSON.stringify(`${jsonSource.href}#other`)}; + createEvidence(target, provider, {result: jsonValueSchema.parse(input)});`, + ), + valid( + "post-fix process capture keeps producer facts", + `${imports} + createEvidence(undefined, provider, { result: capture });`, + ), + valid( + "semantic format admission remains", + `${imports} + createEvidence(target, provider, { result: processCaptureSchema.parse(capture) });`, + ), + valid( + "standalone JSON admission remains", + `${imports} + const observation = { result: jsonValueSchema.parse(result) }; + executeAnalysis(observation);`, + ), + valid( + "strong immutable Evidence contract remains", + `${imports} + createImmutableEvidenceSteps(target, provider, { result: jsonValueSchema.parse(result) });`, + ), + valid( + "digest helper owns its typed JSON input", + `${imports} + const parameters = { ...jsonObjectSchema.parse(scenario), digest: scenarioDigest(scenario) }; + createEvidence(target, provider, { parameters });`, + ), + valid( + "multiuse parsed value retains its separate contract", + `${imports} + const result = jsonValueSchema.parse(input); + consumeJson(result); + createEvidence(target, provider, { result });`, + ), + valid( + "single-use typed local is outside the direct expression rule", + `${imports} + const result: JsonValue = jsonValueSchema.parse(input); + createEvidence(target, provider, { result });`, + ), + valid( + "local creator shadows imported creator", + `${imports} + function build(createEvidence) { + return createEvidence(target, provider, { result: jsonValueSchema.parse(input) }); + }`, + ), + valid( + "local schema shadows imported schema", + `${imports} + function build(jsonValueSchema) { + return createEvidence(target, provider, { result: jsonValueSchema.parse(input) }); + }`, + ), + valid( + "namespace shadowing is respected", + ` + import * as evidence from "../domain/evidence.js"; + import * as json from "../domain/jsonValue.js"; + function build(json) { + return evidence.createEvidence(target, provider, { result: json.jsonValueSchema.parse(input) }); + }`, + ), + valid( + "creator namespace shadowing is respected", + ` + import * as evidence from "../domain/evidence.js"; + import * as json from "../domain/jsonValue.js"; + function build(evidence) { + return evidence.createEvidence(target, provider, { result: json.jsonValueSchema.parse(input) }); + }`, + ), + valid( + "same-named exports from another module are unrelated", + ` + import { createEvidence } from "../other/evidence.js"; + import { jsonValueSchema } from "../domain/jsonValue.js"; + createEvidence(target, provider, { result: jsonValueSchema.parse(input) });`, + ), + valid( + "same-named schema from another module is unrelated", + ` + import { createEvidence } from "../domain/evidence.js"; + import { jsonValueSchema } from "../other/jsonValue.js"; + createEvidence(target, provider, { result: jsonValueSchema.parse(input) });`, + ), + valid( + "safeParse is a separate Result boundary", + `${imports} + createEvidence(target, provider, { result: jsonValueSchema.safeParse(input) });`, + ), + valid( + "nested parsing is not an immediate payload admission", + `${imports} + createEvidence(target, provider, { parameters: { trace_spec: jsonValueSchema.parse(specification) } });`, + ), + valid( + "later spread may replace the parsed field", + `${imports} + createEvidence(target, provider, { result: jsonValueSchema.parse(input), ...overrides });`, + ), + valid( + "later duplicate field replaces the parsed field", + `${imports} + createEvidence(target, provider, { result: jsonValueSchema.parse(input), result: replacement });`, + ), + valid( + "unknown computed field may replace the parsed field", + `${imports} + createEvidence(target, provider, { result: jsonValueSchema.parse(input), [field]: replacement });`, + ), + valid( + "explicit parse options are outside the direct admission rule", + `${imports} + createEvidence(target, provider, { result: jsonValueSchema.parse(input, options) });`, + ), + ], + invalid: [ + invalid( + "file URLs preserve canonical JSON ownership", + `${fileImports} + createEvidence(target, provider, {result: jsonValueSchema.parse(input)});`, + "result", + ), + invalid( + "absolute paths preserve canonical JSON ownership", + `${absoluteImports} + createEvidence(target, provider, {result: jsonValueSchema.parse(input)});`, + "result", + ), + invalid( + "pre-fix process capture repeats JSON admission", + `${imports} + createEvidence(undefined, PROCESS_PROVIDER, { result: jsonValueSchema.parse(capture) });`, + "result", + ), + invalid( + "object parameters repeat generic admission", + `${imports} + createEvidence(target, provider, { parameters: jsonObjectSchema.parse(parameters) });`, + "parameters", + ), + invalid( + "raw payload admission belongs to Evidence", + `${imports} + createEvidence(target, provider, { rawResult: jsonValueSchema.parse(raw) });`, + "rawResult", + ), + invalid( + "aliased imports retain their owner", + ` + import { createEvidence as record } from "../domain/evidence.js"; + import { jsonValueSchema as json } from "../domain/jsonValue.js"; + record(target, provider, { result: json.parse(input) });`, + "result", + ), + invalid( + "namespace members retain their owner", + ` + import * as evidence from "../domain/evidence.js"; + import * as json from "../domain/jsonValue.js"; + evidence.createEvidence(target, provider, { result: json.jsonValueSchema.parse(input) });`, + "result", + ), + invalid( + "static computed members retain their owner", + ` + import * as evidence from "../domain/evidence.js"; + import * as json from "../domain/jsonValue.js"; + evidence["createEvidence"](target, provider, { ["result"]: json["jsonValueSchema"]["parse"](input) });`, + "result", + ), + invalid( + "TypeScript source specifiers resolve to the same owner", + ` + import { createEvidence } from "../domain/evidence.ts"; + import { jsonObjectSchema } from "../domain/jsonValue.ts"; + createEvidence(target, provider, { parameters: jsonObjectSchema.parse(input) });`, + "parameters", + ), + invalid( + "an earlier spread does not replace the final payload", + `${imports} + createEvidence(target, provider, { ...base, result: jsonValueSchema.parse(input) });`, + "result", + ), + invalid( + "unrelated local scopes do not hide the import", + `${imports} + function unrelated(jsonValueSchema) { return jsonValueSchema.parse(input); } + createEvidence(target, provider, { result: jsonValueSchema.parse(input) });`, + "result", + ), + ], +}); diff --git a/tests/evaluation/fixtureRootCleanupRule.test.ts b/tests/evaluation/fixtureRootCleanupRule.test.ts new file mode 100644 index 0000000000..8d6292e84a --- /dev/null +++ b/tests/evaluation/fixtureRootCleanupRule.test.ts @@ -0,0 +1,58 @@ +import { describe, it } from "vitest"; +import { fileURLToPath } from "node:url"; +import { RuleTester } from "oxlint/plugins-dev"; +import rule from "../../scripts/lint/rules/no-fixture-root-cleanup.mjs"; + +RuleTester.describe = describe; +RuleTester.it = it; +const tester = new RuleTester({ + languageOptions: { parserOptions: { lang: "ts" } }, +}); +const filename = fileURLToPath( + new URL("../boundary/ruleControl.test.ts", import.meta.url), +); +const imports = ` +import { createTestTempDirectory as temp } from "../fixtures/temporaryDirectory.js"; +import { afterEach as cleanup, afterAll, onTestFinished } from "vitest"; +import { rm as remove } from "node:fs/promises"; +`; +const fixtureSource = new URL( + "../fixtures/temporaryDirectory.js", + import.meta.url, +); +const localCleanupImports = `import {afterEach} from "vitest"; import {rm} from "node:fs/promises";`; +tester.run("no-fixture-root-cleanup", rule, { + valid: [ + `import {createTestTempDirectory as temp} from ${JSON.stringify(fixtureSource.href)}; ${localCleanupImports} function work(temp: Function) { const root = temp(); afterEach(() => rm(root)); }`, + `import {createTestTempDirectory as temp} from "https://example.test/tests/fixtures/temporaryDirectory.js"; ${localCleanupImports} const root = await temp("owned-"); afterEach(() => rm(root));`, + `import {createTestTempDirectory as temp} from ${JSON.stringify(`${fixtureSource.href}?other`)}; ${localCleanupImports} const root = await temp("owned-"); afterEach(() => rm(root));`, + `${imports} const root = await temp("owned-"); await remove(root, { recursive: true });`, + `${imports} const root = await temp("owned-"); cleanup(() => remove(join(root, "child"), {recursive: true}));`, + `${imports} const root = await otherOwner(); cleanup(() => remove(root, { recursive: true }));`, + `${imports} const root = await temp("owned-"); cleanup(async () => { const root = await otherOwner(); await remove(root, {recursive: true}); });`, + `${imports} function work(temp: Function) { const root = temp(); cleanup(() => remove(root)); }`, + `${imports} const root = await temp("owned-"); function unrelated(cleanup: Function) { cleanup(() => remove(root)); }`, + `${imports} const root = await temp("owned-"); cleanup(() => { function remove(root: string) {} remove(root); });`, + `${imports} const roots = []; const root = await temp("owned-"); roots.push(root); roots.push(await createIdaTarget()); cleanup(async () => { for (const owned of roots) await remove(owned); });`, + `${imports} let root = await temp("owned-"); root = await otherOwner(); cleanup(() => remove(root));`, + `${imports} const root = await temp("owned-"); cleanup(() => connection.close());`, + `${imports} import * as fs from "node:fs/promises"; const root = await temp("owned-"); cleanup((fs: {rm: Function}) => fs.rm(root));`, + `${imports} const root = await temp("owned-"); { const remove = otherOwner; cleanup(() => remove(root)); }`, + 'import { createTestTempDirectory as temp } from "./unrelated.js"; import { afterEach } from "vitest"; import { rm } from "node:fs/promises"; const root = await temp(); afterEach(() => rm(root));', + `${imports} const root = await temp("owned-"); function callback(root: string) { return remove(root); } cleanup(callback);`, + ].map((code) => ({ code, filename })), + invalid: [ + `import {createTestTempDirectory as temp} from ${JSON.stringify(fixtureSource.href)}; ${localCleanupImports} const root = await temp("owned-"); afterEach(() => rm(root));`, + `import {createTestTempDirectory as temp} from ${JSON.stringify(fileURLToPath(fixtureSource))}; ${localCleanupImports} const root = await temp("owned-"); afterEach(() => rm(root));`, + `${imports} const root = await temp("owned-"); cleanup(() => remove(root, {recursive: true}));`, + `${imports} const root = await temp("owned-"); afterAll(async () => { await remove(root, {recursive: true}); });`, + `${imports} const root = await temp("owned-"); onTestFinished(() => remove(root, {recursive: true}));`, + `${imports} const root = await temp("owned-"); const alias = root; cleanup(() => remove(alias));`, + `${imports} const root = await temp("owned-"); async function release() { await remove(root); } cleanup(release);`, + `${imports} const root = await temp("owned-"); const release = () => remove(root); cleanup(release);`, + `${imports} import { rmSync } from "node:fs"; const root = await temp("owned-"); cleanup(() => rmSync(root, {recursive: true}));`, + `${imports} import { removeTestWorkspace as release } from "../support/workspace/workspaceFixture.js"; const root = await temp("owned-"); cleanup(() => release(root));`, + `${imports} const root = await temp("owned-"); cleanup(async () => { await connection.close(); await remove(root, {recursive: true}); });`, + 'import * as fixture from "../fixtures/temporaryDirectory.ts"; import * as vitest from "vitest"; import * as fs from "node:fs/promises"; const root = await fixture.createTestTempDirectory("owned-"); vitest.afterEach(() => fs.rm(root, {recursive: true}));', + ].map((code) => ({ code, filename, errors: [{ messageId: "duplicate" }] })), +}); diff --git a/tests/evaluation/positionalToolContractRule.test.ts b/tests/evaluation/positionalToolContractRule.test.ts new file mode 100644 index 0000000000..231be60622 --- /dev/null +++ b/tests/evaluation/positionalToolContractRule.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from "vitest"; +import { fileURLToPath } from "node:url"; +import { RuleTester } from "oxlint/plugins-dev"; +import rule from "../../scripts/lint/rules/no-positional-tool-contract.mjs"; +import { collectToolCatalogSources } from "../../scripts/lint/rules/tool-catalog-sources.mjs"; + +RuleTester.describe = describe; +RuleTester.it = it; +const tester = new RuleTester({ + languageOptions: { parserOptions: { lang: "ts" } }, +}); +const filename = fileURLToPath( + new URL("../../src/server/ruleControl.ts", import.meta.url), +); +const catalog = + 'import { GPU_TOOL_CONTRACTS as contracts } from "../contracts/gpu/gpuToolContracts.js";'; +const catalogSource = new URL( + "../../src/contracts/gpu/gpuToolContracts.js", + import.meta.url, +); +tester.run("no-positional-tool-contract", rule, { + valid: [ + `import {GPU_TOOL_CONTRACTS as contracts} from ${JSON.stringify(catalogSource.href)}; function read(contracts: {name: string}[]) { return contracts[0].name; }`, + 'import {GPU_TOOL_CONTRACTS as contracts} from "https://example.test/src/contracts/gpu/gpuToolContracts.js"; contracts[0].name;', + `import {GPU_TOOL_CONTRACTS as contracts} from ${JSON.stringify(`${catalogSource.href}#other`)}; contracts[0].name;`, + `${catalog} for (const contract of contracts) register(contract);`, + `${catalog} contracts.forEach(register);`, + `${catalog} const selected = contracts.find(contract => contract.name === "inspect_gpu_code_objects");`, + `${catalog} type Contract = (typeof contracts)[number];`, + `${catalog} const copies = [...contracts];`, + `${catalog} const [...copies] = contracts;`, + `${catalog} function read(contracts: {name: string}[]) { return contracts[0].name; }`, + `${catalog} function read() { const contracts = [{name: "local"}]; return contracts.at(0)?.name; }`, + 'import { GPU_TOOL_CONTRACTS as contracts } from "../unrelated.js"; contracts[0].name;', + 'import { unrelated as contracts } from "../contracts/gpu/gpuToolContracts.js"; contracts[0].name;', + 'const TOOL_CONTRACTS = [{name: "local"}]; TOOL_CONTRACTS[0].name;', + `${catalog} const mutable = [...contracts]; mutable[0].inputSchema;`, + `${catalog} contracts.length;`, + `${catalog} contracts[0]; contracts.at(1); const [unused] = contracts;`, + `${catalog} expect(contracts[0]?.description).toContain("documented behavior");`, + `${catalog} const [first] = contracts; const toggled = contracts.map(contract => contract === first ? {...contract, returnsTargetContent: !contract.returnsTargetContent} : contract); createCatalogDigests(toggled);`, + `${catalog} const selected = contracts[0]; const alias = selected; display(alias.description);`, + `${catalog} const selected = contracts[0]; selected.name = "catalog-control";`, + `${catalog} const [selected] = contracts; const { description } = selected; display(description);`, + `${catalog} function toolRegistrationOptions(contract: unknown) {} toolRegistrationOptions(contracts[0]);`, + `${catalog} { const contracts = [{name: "local"}]; contracts[0].name; }`, + 'import * as catalogs from "../contracts/gpu/gpuToolContracts.js"; function read(catalogs: {GPU_TOOL_CONTRACTS: {name: string}[]}) { return catalogs.GPU_TOOL_CONTRACTS[0].name; }', + 'import * as catalogs from "../contracts/gpu/gpuToolContracts.js"; catalogs.other[0].name;', + ].map((code) => ({ code, filename })), + invalid: [ + `import {GPU_TOOL_CONTRACTS as contracts} from ${JSON.stringify(catalogSource.href)}; contracts[0].name;`, + `import {GPU_TOOL_CONTRACTS as contracts} from ${JSON.stringify(fileURLToPath(catalogSource))}; contracts[0].inputSchema;`, + `${catalog} import {toolRegistrationOptions as options} from ${JSON.stringify(new URL("../../src/server/toolRegistrationOptions.js", import.meta.url).href)}; options(contracts[0]);`, + `${catalog} register(contracts[0].name);`, + `${catalog} contracts["0"].inputSchema.parse(input);`, + `${catalog} contracts.at(-1)?.outputSchema;`, + `${catalog} contracts["at"](1)?.name;`, + `${catalog} const alias = contracts; alias[2].name;`, + `${catalog} const first = contracts; const second = first; second[0].outputSchema;`, + `${catalog} type Selected = (typeof contracts)[0];`, + `${catalog} const [first] = contracts; register(first.name);`, + `${catalog} const [, second] = contracts; const selected = second; selected.inputSchema;`, + `${catalog} const selected = contracts[0]; const alias = selected; alias.inputSchema;`, + `${catalog} const { inputSchema } = contracts[0];`, + `${catalog} import {toolRegistrationOptions as options} from "./toolRegistrationOptions.js"; options(contracts[0]);`, + `${catalog} import {applicationToolRegistrationOptions as options} from "./toolRegistrationOptions.ts"; const selected = contracts.at(0); options(selected);`, + `${catalog} const selected = contracts[0]; selected.name; selected.outputSchema;`, + 'import * as catalogs from "../contracts/gpu/gpuToolContracts.js"; catalogs.GPU_TOOL_CONTRACTS[0].name;', + 'import * as catalogs from "../contracts/gpu/gpuToolContracts.js"; type Selected = (typeof catalogs.GPU_TOOL_CONTRACTS)[0];', + 'import { TOOL_CONTRACTS } from "../contracts/toolContracts.ts"; TOOL_CONTRACTS[0].name;', + 'import { MODEL_TOOL_CONTRACTS as selected } from "../contracts/model/modelToolContracts.js"; selected[0].inputSchema;', + 'import { IL2CPP_TOOL_CONTRACTS as selected } from "../contracts/il2cpp/il2cppToolContracts.js"; selected[0].outputSchema;', + 'import { BROWSER_SCENARIO_TOOL_CONTRACTS as nested } from "../contracts/browserScenarioToolContracts.js"; nested[0].name;', + 'import { FUNCTION_WORKFLOW_TOOL_CONTRACTS as nested } from "../contracts/functionWorkflowToolContracts.js"; nested.at(0)?.name;', + ].map((code) => ({ code, filename, errors: [{ messageId: "positional" }] })), +}); + +const sourcePath = (name: string) => + fileURLToPath(new URL(`./catalog-rule-fixture/${name}`, import.meta.url)); + +describe("canonical catalog source composition", () => { + it("discovers newly composed exports and nested groups without evaluating modules", () => { + const sources = new Map([ + [ + sourcePath("toolContracts.ts"), + ` + import { NEW_CATALOG as renamed } from "./newCatalog.js"; + import { GROUP } from "./group.js"; + import { UNUSED } from "./unused.js"; + const generated = buildContracts(); + const local = [...renamed, ...GROUP, ...generated, ...UNUSED.map(makeContract)]; + const alias = local; + export const TOOL_CONTRACTS = [...alias] as const; + `, + ], + [ + sourcePath("newCatalog.ts"), + ` + throw new Error("Must never execute the runtime catalog"); + export const NEW_CATALOG = [{name: "new_tool"}]; + `, + ], + [ + sourcePath("group.ts"), + ` + import { NESTED as alias } from "./nested.ts"; + export const GROUP = [...alias] satisfies readonly unknown[]; + `, + ], + [ + sourcePath("nested.ts"), + 'export const NESTED = [{name: "nested_tool"}];', + ], + ]); + const actual = collectToolCatalogSources( + sourcePath("toolContracts.ts"), + (path) => { + const source = sources.get(path); + if (source === undefined) + throw new Error(`Unexpected source read: ${path}`); + return source; + }, + ); + expect(actual).toEqual( + new Map([ + [sourcePath("toolContracts"), new Set(["TOOL_CONTRACTS"])], + [sourcePath("newCatalog"), new Set(["NEW_CATALOG"])], + [sourcePath("group"), new Set(["GROUP"])], + [sourcePath("nested"), new Set(["NESTED"])], + ]), + ); + }); + + it("accepts official computed definitions while discovering imported catalog groups", () => { + const actual = collectToolCatalogSources( + fileURLToPath( + new URL("../../src/contracts/toolContracts.ts", import.meta.url), + ), + ); + const official = fileURLToPath( + new URL("../../src/contracts/officialToolContracts", import.meta.url), + ); + const definitions = fileURLToPath( + new URL("../../src/contracts/hopperMemoryContracts", import.meta.url), + ); + const nested = fileURLToPath( + new URL( + "../../src/contracts/browserScenarioToolContracts", + import.meta.url, + ), + ); + expect(actual.get(official)).toEqual(new Set(["OFFICIAL_TOOL_CONTRACTS"])); + expect(actual.has(definitions)).toBe(false); + expect(actual.get(nested)).toEqual( + new Set(["BROWSER_SCENARIO_TOOL_CONTRACTS"]), + ); + }); + + it("propagates source-read and parse failures instead of disabling enforcement", () => { + const failure = new Error("catalog source unavailable"); + expect(() => + collectToolCatalogSources(sourcePath("toolContracts.ts"), () => { + throw failure; + }), + ).toThrow(failure); + expect(() => + collectToolCatalogSources( + sourcePath("toolContracts.ts"), + () => "export const TOOL_CONTRACTS = [", + ), + ).toThrow(); + }); + + it.each([ + "export const TOOL_CONTRACTS = buildCatalog();", + "export const OTHER = [];", + "const alias = alias; export const TOOL_CONTRACTS = [...alias];", + "export const TOOL_CONTRACTS = [...unresolved];", + ])("refuses unsupported or missing canonical composition: %s", (source) => { + expect(() => + collectToolCatalogSources(sourcePath("toolContracts.ts"), () => source), + ).toThrow(); + }); +}); diff --git a/tests/fixtures/artifactEvidence.ts b/tests/fixtures/artifactEvidence.ts index 57f50c9a58..49c748ed13 100644 --- a/tests/fixtures/artifactEvidence.ts +++ b/tests/fixtures/artifactEvidence.ts @@ -1,6 +1,5 @@ import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; import { createEvidence, type Evidence } from "../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../src/domain/jsonValue.js"; /** Partition a real scan into authenticated partial observations for merge tests. */ export const fragmentInventoryEvidence = ( @@ -26,7 +25,7 @@ export const fragmentInventoryEvidence = ( parameters: evidence.parameters, confidence: "derived", authority: evidence.authority, - result: jsonValueSchema.parse({ + result: { ...inventory, nodes: inventory.nodes.filter( (_, ordinal) => ordinal % count === index, @@ -37,7 +36,7 @@ export const fragmentInventoryEvidence = ( edges: inventory.edges.filter( (_, ordinal) => ordinal % count === index, ), - }), + }, }, ), ); diff --git a/tests/fixtures/nativeFunctionMatching.ts b/tests/fixtures/nativeFunctionMatching.ts index b5969f1fb6..bc0bf80b61 100644 --- a/tests/fixtures/nativeFunctionMatching.ts +++ b/tests/fixtures/nativeFunctionMatching.ts @@ -1,5 +1,5 @@ import { createEvidence, type Evidence } from "../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../src/domain/jsonValue.js"; + import { FUNCTION_COMPARISON_EXAMPLE } from "../../src/contracts/functionComparisonExample.js"; import type { NativeSymbolMetadata } from "../../src/domain/native/nativeSymbol.js"; import { functionDossierSchema } from "../../src/domain/hopperValues.js"; @@ -24,7 +24,7 @@ export const nativeMatchingInventory = ( createEvidence(nativeMatchingTarget(side), nativeMatchingProvider, { operation: "list_procedures", parameters: {}, - result: jsonValueSchema.parse(entries), + result: entries, evidenceLinks: [...evidenceLinks], }); /** Explicit imported-origin fixture for tests that select exact-symbol correspondence. */ @@ -97,7 +97,7 @@ export const nativeMatchingDetails = ( observation = createEvidence(target, nativeMatchingProvider, { operation: "analyze_function", parameters: { procedure: address }, - result: jsonValueSchema.parse(value), + result: value, relationshipCoverage: { status: "complete" }, }); } else diff --git a/tests/fixtures/oxlintConfiguration.ts b/tests/fixtures/oxlintConfiguration.ts new file mode 100644 index 0000000000..0e9cabbce2 --- /dev/null +++ b/tests/fixtures/oxlintConfiguration.ts @@ -0,0 +1,39 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; + +const repository = fileURLToPath(new URL("../../", import.meta.url)); +const configurationSchema = z.looseObject({ + jsPlugins: z + .array(z.union([z.string(), z.looseObject({ specifier: z.string() })])) + .optional(), +}); + +/** Preserve repository lint policy while resolving local plugins at their owner. */ +export const writeRepositoryOxlintConfiguration = async ( + project: string, +): Promise => { + const configuration = configurationSchema.parse( + JSON.parse(await readFile(join(repository, ".oxlintrc.json"), "utf8")), + ); + const rebase = (specifier: string): string => + specifier.startsWith(".") ? resolve(repository, specifier) : specifier; + const path = join(project, ".oxlintrc.json"); + await writeFile( + path, + JSON.stringify({ + ...configuration, + ...(configuration.jsPlugins === undefined + ? {} + : { + jsPlugins: configuration.jsPlugins.map((plugin) => + typeof plugin === "string" + ? rebase(plugin) + : { ...plugin, specifier: rebase(plugin.specifier) }, + ), + }), + }), + ); + return path; +}; diff --git a/tests/fixtures/reconstructionProofApplicability.ts b/tests/fixtures/reconstructionProofApplicability.ts index 15d3882afc..c1f71e1761 100644 --- a/tests/fixtures/reconstructionProofApplicability.ts +++ b/tests/fixtures/reconstructionProofApplicability.ts @@ -1,7 +1,7 @@ import { readyReviewedLedgerRequest } from "../../src/application/ReconstructionObligationLedger.fixture.js"; import { createEvidence } from "../../src/domain/evidence.js"; import { createEvidenceBundle } from "../../src/domain/evidenceBundle.js"; -import { jsonValueSchema } from "../../src/domain/jsonValue.js"; + import { reconstructionObligationProofSchema } from "../../src/domain/reconstructionObligationProof.js"; import type { ReconstructionObligationLedgerInput } from "../../src/domain/reconstructionObligationLedgerSchemas.js"; @@ -16,7 +16,7 @@ const replaceProof = ( predicateType: original.predicate_type, operation: original.operation, parameters: original.parameters, - result: jsonValueSchema.parse(result), + result: result, confidence: original.confidence, authority: original.authority, }); diff --git a/tests/fixtures/webScriptCapture.ts b/tests/fixtures/webScriptCapture.ts index fef099676a..06682ef614 100644 --- a/tests/fixtures/webScriptCapture.ts +++ b/tests/fixtures/webScriptCapture.ts @@ -5,7 +5,6 @@ import { browserScenarioCaptureSchema } from "../../src/domain/browserScenarioCa import { DEFAULT_BROWSER_SCENARIO_CAPTURE_LIMITS } from "../../src/domain/browserScenarioCaptureLimits.js"; import { browserScenarioEventSchema } from "../../src/domain/browserScenarioCaptureValues.js"; import { createEvidence } from "../../src/domain/evidence.js"; -import { jsonValueSchema } from "../../src/domain/jsonValue.js"; /** Source-owned script capture with independently authored bytes. */ export const scriptScenarioFixture = ( @@ -134,7 +133,7 @@ export const scriptCaptureEvidenceFixture = () => { id: "source-fixture", name: "Source fixture", version: "1" }, { operation: "capture_browser_scenario", - result: jsonValueSchema.parse(scriptScenarioFixture()), + result: scriptScenarioFixture(), parameters: {}, confidence: "observed", authority: "controlled-replay", From 42c16ba6315a7a6ba52a06e74a20eb1671eaab21 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:18:09 +0800 Subject: [PATCH 32/37] fix(tests): resolve lint plugins in boundary regression --- tests/evaluation/moduleBoundaries.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/evaluation/moduleBoundaries.test.ts b/tests/evaluation/moduleBoundaries.test.ts index 786f1032ae..15e6c0d0d7 100644 --- a/tests/evaluation/moduleBoundaries.test.ts +++ b/tests/evaluation/moduleBoundaries.test.ts @@ -1,5 +1,5 @@ import { execFile, spawnSync } from "node:child_process"; -import { copyFile, mkdir, rename, writeFile } from "node:fs/promises"; +import { mkdir, rename, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; @@ -7,6 +7,7 @@ import { describe, expect, it } from "vitest"; import { inspectModuleBoundaries } from "../../scripts/lib/module-boundaries.mjs"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { writeRepositoryOxlintConfiguration } from "../fixtures/oxlintConfiguration.js"; describe("process capture implementation ownership", () => { it.each([ @@ -329,8 +330,7 @@ describe("test lane import boundaries", () => { it("checks the test-lane import matrix in one lint invocation", async () => { const temporary = await createTestTempDirectory("rea-test-lane-"); - const config = join(temporary, ".oxlintrc.json"); - await copyFile(resolve(".oxlintrc.json"), config); + const config = await writeRepositoryOxlintConfiguration(temporary); for (const [file, dependency] of cases) { await mkdir(dirname(join(temporary, file)), { recursive: true }); await writeFile( @@ -344,6 +344,8 @@ describe("test lane import boundaries", () => { resolve("node_modules/oxlint/bin/oxlint"), "-c", config, + "--threads", + "1", "--format", "json", ...cases.map(([file]) => file), @@ -351,6 +353,10 @@ describe("test lane import boundaries", () => { { cwd: temporary, encoding: "utf8" }, ); if (result.error !== undefined) throw result.error; + expect( + result.stdout.trimStart().startsWith("{"), + `oxlint exited ${String(result.status)} (${String(result.signal)}): ${result.stdout}\n${result.stderr}`, + ).toBe(true); const report = JSON.parse(result.stdout) as { diagnostics: Array<{ filename: string; code: string }>; }; From 760d9f6019f733aca88b3a819561a3dea78a1fc9 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:20:37 +0800 Subject: [PATCH 33/37] fix(source-maps): retain primary failures through cleanup retries --- .../sourceMaps/SourceMapCommand.test.ts | 78 ++++++++++++++++++- src/javascript/sourceMaps/SourceMapCommand.ts | 46 +++++++---- .../sourceMaps/SourceMapDecoder.test.ts | 70 ++++++++++++----- src/javascript/sourceMaps/SourceMapDecoder.ts | 14 ++-- .../javascript/sourceMapPreparation.test.ts | 28 +++++-- .../sourceMapRuntimeAvailability.test.ts | 31 +++++--- 6 files changed, 207 insertions(+), 60 deletions(-) diff --git a/src/javascript/sourceMaps/SourceMapCommand.test.ts b/src/javascript/sourceMaps/SourceMapCommand.test.ts index cdca518743..ae2b338408 100644 --- a/src/javascript/sourceMaps/SourceMapCommand.test.ts +++ b/src/javascript/sourceMaps/SourceMapCommand.test.ts @@ -1,5 +1,5 @@ import { access } from "node:fs/promises"; -import { expect, it, vi } from "vitest"; +import { afterEach, expect, it, vi } from "vitest"; import { AnalysisCancelledError, AnalysisResourceConstraintError, @@ -14,8 +14,18 @@ import { WEB_SOURCE_MAP_LIMITS } from "../../domain/webSourceLocation.js"; import { runSourceMapCommand, SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, + type SourceMapCleanupRetention, } from "./SourceMapCommand.js"; +const unexpectedCleanup: SourceMapCleanupRetention = {}; +afterEach(async () => { + const owner = unexpectedCleanup.owner; + const failure = await owner?.close(); + if (failure !== undefined) throw failure; + unexpectedCleanup.owner = undefined; + expect(owner).toBeUndefined(); +}); + const runChild = ( signal: AbortSignal | undefined, script: string, @@ -44,6 +54,7 @@ const runChild = ( return child; }, }, + unexpectedCleanup, ); const exitsAfterHoldingOutput = @@ -65,6 +76,7 @@ it("preserves the exact object reason for an already-aborted caller", async () = throw new Error("must not launch"); }, }, + unexpectedCleanup, ); expect(result.ok).toBe(false); if (result.ok) throw new Error("expected cancellation"); @@ -72,6 +84,67 @@ it("preserves the exact object reason for an already-aborted caller", async () = expect(result.error.cause).toBe(reason); }); +it.each([null, { caller: "late source-map cancellation" }])( + "retains a late caller reason %j when runtime cleanup first fails", + async (reason) => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-source-map-late-cleanup-test-", + }); + const controller = new AbortController(); + let cleanupAttempts = 0; + const retainedCleanup: SourceMapCleanupRetention = {}; + try { + const result = await runSourceMapCommand( + "{}", + "/probe/source.map", + { signal: controller.signal }, + { + prepareOwnership: async () => {}, + createRuntime: async () => ({ + path: root.path, + close: async () => { + cleanupAttempts += 1; + if (cleanupAttempts === 1) { + controller.abort(reason); + throw new Error("controlled late cleanup failure"); + } + await root.close(); + }, + }), + launcher: async (options) => + spawnOwnedProviderProcess({ + ...options, + arguments: ["-e", 'process.stdout.write("complete")'], + }), + }, + retainedCleanup, + ); + expect(result.ok).toBe(false); + if (result.ok) + throw new Error("expected cancellation with cleanup failure"); + expect(result.error.cleanupPrimary).toBeInstanceOf( + AnalysisCancelledError, + ); + expect(result.error.cleanupPrimary?.cause).toBe(reason); + expect(result.error.cleanupResources).toEqual([root.path]); + expect(result.error.cleanupIncomplete).toBe(true); + expect(retainedCleanup.owner).toBeDefined(); + await retainedCleanup.owner?.close(); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cleanupResources).toEqual([]); + expect(result.error.cleanupPrimary?.cause).toBe(reason); + } finally { + await retainedCleanup.owner?.close(); + await root.close(); + } + }, +); + +it("returns completed codec output after successful cleanup", async () => { + const result = await runChild(undefined, 'process.stdout.write("complete")'); + expect(result).toEqual({ ok: true, value: "complete" }); +}); + it("preserves exact caller reason when cancellation precedes process exit", async () => { const reason = { caller: "active source-map cancellation" }; const controller = new AbortController(); @@ -222,6 +295,7 @@ it("preserves an observed process-memory limit over the codec exit status", asyn return spawned; }, }, + unexpectedCleanup, ); } finally { waitSpy.mockRestore(); @@ -266,6 +340,7 @@ it("keeps a timeout distinct from caller cancellation", async () => { throw new Error("launcher settled after deadline"); }, }, + unexpectedCleanup, ); expect(result.ok).toBe(false); if (result.ok) throw new Error("expected timeout"); @@ -301,6 +376,7 @@ it("applies a non-default decode deadline and releases its runtime", async () => throw new Error("launcher settled after deadline"); }, }, + unexpectedCleanup, 250, ); expect(result.ok).toBe(false); diff --git a/src/javascript/sourceMaps/SourceMapCommand.ts b/src/javascript/sourceMaps/SourceMapCommand.ts index 3cf7485560..3da5b9eef6 100644 --- a/src/javascript/sourceMaps/SourceMapCommand.ts +++ b/src/javascript/sourceMaps/SourceMapCommand.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; import { AnalysisCancelledError, AnalysisOutputError, @@ -62,9 +63,12 @@ export interface SourceMapDecoderDependencies { } export interface SourceMapCleanupOwner { - close( - previousError: string | null, - ): Promise; + close(): Promise; +} + +/** The caller retains retry authority independently of the returned error. */ +export interface SourceMapCleanupRetention { + owner?: SourceMapCleanupOwner | undefined; } type CodecPhysicalObservation = @@ -88,7 +92,6 @@ type CodecOutcome = export class SourceMapCleanupFailure extends ProviderCleanupError { constructor( - readonly cleanupOwner: SourceMapCleanupOwner, resources: readonly string[], diagnostics: Readonly>, ) { @@ -102,6 +105,7 @@ export const runSourceMapCommand = async ( path: string, options: ExecutionOptions | undefined, dependencies: SourceMapDecoderDependencies, + retainedCleanup: SourceMapCleanupRetention, decodeTimeoutMs: number = DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, ): Promise> => { try { @@ -141,13 +145,26 @@ export const runSourceMapCommand = async ( command.dispose(); deadline.dispose(); } - const cleanup = await command.close(result.ok ? null : result.error.message); - if (cleanup !== undefined) return err(cleanup); - return result.ok && options?.signal?.aborted - ? err( - new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), - ) - : result; + const cleanup = await command.close(); + const completed = + result.ok && options?.signal?.aborted + ? err( + new AnalysisCancelledError(OPERATION, { + cause: options.signal.reason, + }), + ) + : result; + if (cleanup === undefined) return completed; + retainedCleanup.owner = command; + return completed.ok + ? err(cleanup) + : err( + analysisErrorWithCleanupFailure( + completed.error, + cleanup, + OPERATION, + ).bindCleanupDependencies([cleanup], command), + ); }; class OwnedCodec implements SourceMapCleanupOwner { @@ -471,9 +488,7 @@ class OwnedCodec implements SourceMapCleanupOwner { ]; } - async close( - previousError: string | null, - ): Promise { + async close(): Promise { const failures: string[] = []; let processReleased = this.#supervisor === undefined; if (this.#supervisor !== undefined) { @@ -497,10 +512,9 @@ class OwnedCodec implements SourceMapCleanupOwner { this.#cleanupAnchor = undefined; return undefined; } - const failure = new SourceMapCleanupFailure(this, this.cleanupResources, { + const failure = new SourceMapCleanupFailure(this.cleanupResources, { map_path: this.path, cleanup_failures: failures, - previous_error: previousError, codec_pid: this.#supervisor?.launch.process.pid ?? null, }).bindCleanupResourceOwner(this); if (this.#cleanupAnchor === undefined) this.#cleanupAnchor = failure; diff --git a/src/javascript/sourceMaps/SourceMapDecoder.test.ts b/src/javascript/sourceMaps/SourceMapDecoder.test.ts index dc062ac01b..9236716b0f 100644 --- a/src/javascript/sourceMaps/SourceMapDecoder.test.ts +++ b/src/javascript/sourceMaps/SourceMapDecoder.test.ts @@ -67,19 +67,30 @@ it("retains failed cleanup ownership and retries it before another decode", asyn expect(failed.ok).toBe(false); if (failed.ok) throw new Error("expected physical failure with cleanup failure"); - expect(failed.error).toBeInstanceOf(SourceMapCleanupFailure); - if (!(failed.error instanceof SourceMapCleanupFailure)) - throw new Error("expected source-map cleanup ownership"); - expect(failed.error.cleanupOwner).toBeDefined(); + const primary = failed.error.cleanupPrimary; + expect(primary).toMatchObject({ + _tag: "ProviderAdapterError", + diagnostics: { + exit_code: 7, + stderr: "physical exit 7", + }, + }); + expect(primary?.capturedOutput?.stderr).toBe("physical exit 7"); + expect(failed.error.capturedOutput).toEqual(primary?.capturedOutput); + expect(failed.error.cause).toBeInstanceOf(AggregateError); + expect((failed.error.cause as AggregateError).errors[0]).toBe(primary); expect(failed.error.cleanupResources).toEqual([root.path]); expect(failed.error.diagnostics).toMatchObject({ - cleanup_failures: ["injected root cleanup failure"], - }); - expect(failed.error.diagnostics).toMatchObject({ - previous_error: expect.stringContaining( - "source-map-decoder adapter failed", - ), + primary_error: { code: "execution_failure" }, + cleanup_error: { + details: { + diagnostics: { + cleanup_failures: ["injected root cleanup failure"], + }, + }, + }, }); + expect(projectAnalysisError(failed.error).code).toBe("cleanup_incomplete"); await expect(access(root.path)).resolves.toBeUndefined(); const invalidInput = { ...input, text: null } as unknown as typeof input; @@ -92,11 +103,16 @@ it("retains failed cleanup ownership and retries it before another decode", asyn await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); expect(failed.error.cleanupIncomplete).toBe(false); expect(failed.error.cleanupResources).toEqual([]); - expect(projectAnalysisError(failed.error).code).not.toBe( - "cleanup_incomplete", - ); + expect(projectAnalysisError(failed.error).code).toBe("execution_failure"); + expect(failed.error.cleanupPrimary).toBe(primary); expect(failed.error.diagnostics).toMatchObject({ - cleanup_failures: ["injected root cleanup failure"], + cleanup_error: { + details: { + diagnostics: { + cleanup_failures: ["injected root cleanup failure"], + }, + }, + }, }); } finally { await decoder.close(); @@ -108,6 +124,7 @@ it("settles every source-map cleanup failure when final close releases the exact prefix: "rea-source-map-final-cleanup-test-", }); let attempts = 0; + const launchCause = new Error("controlled launch failure"); const decoder = new SourceMapDecoder({ prepareOwnership: async () => {}, createRuntime: async () => ({ @@ -120,12 +137,16 @@ it("settles every source-map cleanup failure when final close releases the exact }, }), launcher: async () => { - throw new Error("controlled launch failure"); + throw launchCause; }, }); try { const initial = await decoder.trace(input); if (initial.ok) throw new Error("expected initial cleanup failure"); + expect(initial.error.cleanupPrimary?.cause).toBe(launchCause); + expect((initial.error.cause as AggregateError).errors[0]).toBe( + initial.error.cleanupPrimary, + ); const retry = await decoder.trace(input); if (retry.ok) throw new Error("expected retry cleanup failure"); for (const failure of [initial.error, retry.error]) { @@ -139,11 +160,22 @@ it("settles every source-map cleanup failure when final close releases the exact for (const failure of [initial.error, retry.error]) { expect(failure.cleanupIncomplete).toBe(false); expect(failure.cleanupResources).toEqual([]); - expect(projectAnalysisError(failure).code).not.toBe("cleanup_incomplete"); - expect(failure.diagnostics?.cleanup_failures).toEqual([ - expect.stringContaining("runtime cleanup attempt"), - ]); + expect(projectAnalysisError(failure).code).toBe("execution_failure"); } + expect(initial.error.cleanupPrimary?.cause).toBe(launchCause); + expect(initial.error.diagnostics).toMatchObject({ + cleanup_error: { + details: { + diagnostics: { + cleanup_failures: ["runtime cleanup attempt 1 failed"], + }, + }, + }, + }); + expect(retry.error).toBeInstanceOf(SourceMapCleanupFailure); + expect(retry.error.diagnostics?.cleanup_failures).toEqual([ + "runtime cleanup attempt 2 failed", + ]); } finally { await decoder.close(); } diff --git a/src/javascript/sourceMaps/SourceMapDecoder.ts b/src/javascript/sourceMaps/SourceMapDecoder.ts index 4159330ea1..08eca6a1c5 100644 --- a/src/javascript/sourceMaps/SourceMapDecoder.ts +++ b/src/javascript/sourceMaps/SourceMapDecoder.ts @@ -22,9 +22,9 @@ import { } from "./SourceMapCodecProtocol.js"; import { SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, - SourceMapCleanupFailure, runSourceMapCommand, type SourceMapCleanupOwner, + type SourceMapCleanupRetention, type SourceMapDecoderDependencies, } from "./SourceMapCommand.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; @@ -114,18 +114,18 @@ export class SourceMapDecoder implements WebSourceMapPort { new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), ); const serialized = encoded.join(""); + const retainedCleanup: SourceMapCleanupRetention = {}; const response = await runSourceMapCommand( serialized, input.path, options, this.dependencies, + retainedCleanup, input.decodeTimeoutMs ?? DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, ); - if (!response.ok) { - if (response.error instanceof SourceMapCleanupFailure) - this.#pendingCleanup = response.error.cleanupOwner; - return response; - } + if (retainedCleanup.owner !== undefined) + this.#pendingCleanup = retainedCleanup.owner; + if (!response.ok) return response; let value: unknown; try { value = JSON.parse(response.value); @@ -174,7 +174,7 @@ export class SourceMapDecoder implements WebSourceMapPort { async #retryCleanup(): Promise { const pending = this.#pendingCleanup; if (pending === undefined) return undefined; - const failure = await pending.close(null); + const failure = await pending.close(); if (failure !== undefined) return failure; this.#pendingCleanup = undefined; return undefined; diff --git a/tests/boundary/javascript/sourceMapPreparation.test.ts b/tests/boundary/javascript/sourceMapPreparation.test.ts index 5d9b01ff05..904462079f 100644 --- a/tests/boundary/javascript/sourceMapPreparation.test.ts +++ b/tests/boundary/javascript/sourceMapPreparation.test.ts @@ -2,7 +2,10 @@ import { expect, it, vi } from "vitest"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; -import { runSourceMapCommand } from "../../../src/javascript/sourceMaps/SourceMapCommand.js"; +import { + runSourceMapCommand, + type SourceMapCleanupRetention, +} from "../../../src/javascript/sourceMaps/SourceMapCommand.js"; it.each([ null, @@ -11,6 +14,7 @@ it.each([ "preserves opaque cancellation during ownership preparation: %s", async (reason) => { const controller = new AbortController(); + const retainedCleanup: SourceMapCleanupRetention = {}; const createRuntime = vi.fn(async () => { throw new Error("Cancelled preparation must not acquire a runtime"); }); @@ -29,7 +33,9 @@ it.each([ createRuntime, launcher, }, + retainedCleanup, ); + expect(retainedCleanup.owner).toBeUndefined(); if (result.ok) throw new Error("Preparation must return cancellation"); expect(result.error).toBeInstanceOf(AnalysisCancelledError); expect(result.error.cause).toBe(reason); @@ -46,19 +52,27 @@ it.each([ it("retains diagnostics for an unknown ownership preparation failure", async () => { const cause = new Error("unexpected ownership failure"); + const retainedCleanup: SourceMapCleanupRetention = {}; const createRuntime = vi.fn(async () => { throw new Error("Failed preparation must not acquire a runtime"); }); const launcher = vi.fn(async () => { throw new Error("Failed preparation must not launch a codec"); }); - const result = await runSourceMapCommand("{}", "/selected.map", undefined, { - prepareOwnership: async () => { - throw cause; + const result = await runSourceMapCommand( + "{}", + "/selected.map", + undefined, + { + prepareOwnership: async () => { + throw cause; + }, + createRuntime, + launcher, }, - createRuntime, - launcher, - }); + retainedCleanup, + ); + expect(retainedCleanup.owner).toBeUndefined(); if (result.ok) throw new Error("Preparation must return its failure"); expect(result.error).toBeInstanceOf(ProviderAdapterError); expect(result.error.cause).toBe(cause); diff --git a/tests/boundary/javascript/sourceMapRuntimeAvailability.test.ts b/tests/boundary/javascript/sourceMapRuntimeAvailability.test.ts index 753a3bd470..19970876f7 100644 --- a/tests/boundary/javascript/sourceMapRuntimeAvailability.test.ts +++ b/tests/boundary/javascript/sourceMapRuntimeAvailability.test.ts @@ -1,6 +1,9 @@ import { expect, it } from "vitest"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; -import { runSourceMapCommand } from "../../../src/javascript/sourceMaps/SourceMapCommand.js"; +import { + runSourceMapCommand, + type SourceMapCleanupRetention, +} from "../../../src/javascript/sourceMaps/SourceMapCommand.js"; import { PrivateRuntimeRootUnavailableError } from "../../../src/process/PrivateRuntimeRootUnavailableError.js"; it.each([false, true])( @@ -8,16 +11,24 @@ it.each([false, true])( async (retryable) => { const reason = "Native runtime self-test failed with a sharing violation"; let launches = 0; - const result = await runSourceMapCommand("{}", "/selected.map", undefined, { - prepareOwnership: async () => {}, - createRuntime: async () => { - throw new PrivateRuntimeRootUnavailableError(reason, { retryable }); + const retainedCleanup: SourceMapCleanupRetention = {}; + const result = await runSourceMapCommand( + "{}", + "/selected.map", + undefined, + { + prepareOwnership: async () => {}, + createRuntime: async () => { + throw new PrivateRuntimeRootUnavailableError(reason, { retryable }); + }, + launcher: async () => { + launches += 1; + throw new Error("A refused runtime must not launch a codec"); + }, }, - launcher: async () => { - launches += 1; - throw new Error("A refused runtime must not launch a codec"); - }, - }); + retainedCleanup, + ); + expect(retainedCleanup.owner).toBeUndefined(); expect(launches).toBe(0); if (result.ok) throw new Error("Unavailable runtime must refuse decoding"); expect(projectAnalysisError(result.error)).toMatchObject({ From be698d4761bf9ad86e730e40e7a645f2607190b3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:20:52 +0800 Subject: [PATCH 34/37] test: verify assertion contracts and the production evaluation gate --- scripts/lib/agent-evaluation-report-gate.mjs | 18 +++ scripts/verify-agent-experience.mjs | 9 +- src/config/bridgeMaxRequestBytes.test.ts | 14 +- .../sourceMaps/SourceMapDecoder.test.ts | 8 +- .../model-evals/agentEvaluationReport.test.ts | 124 +++++------------- 5 files changed, 69 insertions(+), 104 deletions(-) create mode 100644 scripts/lib/agent-evaluation-report-gate.mjs diff --git a/scripts/lib/agent-evaluation-report-gate.mjs b/scripts/lib/agent-evaluation-report-gate.mjs new file mode 100644 index 0000000000..1ae5c705fa --- /dev/null +++ b/scripts/lib/agent-evaluation-report-gate.mjs @@ -0,0 +1,18 @@ +import { tsImport } from "tsx/esm/api"; + +const { agentEvaluationPassed } = await tsImport( + new URL( + "../../tests/evaluation/model-evals/AgentEvaluationReport.ts", + import.meta.url, + ).href, + import.meta.url, +); + +/** Fail the runner after publishing a report with any rejected scenario. */ +export const assertAgentEvaluationReportPassed = (results) => { + const failed = results.filter((result) => !agentEvaluationPassed(result)); + if (failed.length > 0) + throw new Error( + `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`, + ); +}; diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index a6b0fdc480..a9923e4e61 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -15,6 +15,7 @@ import { tsImport } from "tsx/esm/api"; import { PRODUCT_IDENTITY } from "../dist/identity.js"; import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs"; +import { assertAgentEvaluationReportPassed } from "./lib/agent-evaluation-report-gate.mjs"; import { closeVerifierResources, withVerifierCleanup, @@ -44,7 +45,7 @@ const { ).href, import.meta.url, ); -const { agentEvaluationPassed, summarizeFactualCorrectness } = await tsImport( +const { summarizeFactualCorrectness } = await tsImport( new URL( "../tests/evaluation/model-evals/AgentEvaluationReport.ts", import.meta.url, @@ -413,11 +414,7 @@ await withVerifierCleanup( await writeFile(resolve(reportPath), encoded); } - const failed = results.filter((result) => !agentEvaluationPassed(result)); - if (failed.length > 0) - throw new Error( - `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`, - ); + assertAgentEvaluationReportPassed(results); }, // Even a deliberately retained fixture directory must never retain auth. [ diff --git a/src/config/bridgeMaxRequestBytes.test.ts b/src/config/bridgeMaxRequestBytes.test.ts index 1d931cc395..1037bb79fb 100644 --- a/src/config/bridgeMaxRequestBytes.test.ts +++ b/src/config/bridgeMaxRequestBytes.test.ts @@ -142,6 +142,16 @@ it.each([ const doctor = parseEnvironment({ [BRIDGE_MAX_REQUEST_BYTES_SETTING]: value, }); - expect(startup.ok).toBe(false); - expect(doctor.ok).toBe(false); + if (startup.ok) throw new Error(`Startup accepted ${JSON.stringify(value)}`); + expect(startup.error._tag).toBe("ConfigurationError"); + expect(startup.error.settings).toContainEqual({ + setting: BRIDGE_MAX_REQUEST_BYTES_SETTING, + constraint: bridgeMaxRequestBytesConstraint, + }); + if (doctor.ok) throw new Error(`Doctor accepted ${JSON.stringify(value)}`); + expect(doctor.error._tag).toBe("ConfigurationError"); + expect(doctor.error.settings).toContainEqual({ + setting: BRIDGE_MAX_REQUEST_BYTES_SETTING, + constraint: bridgeMaxRequestBytesConstraint, + }); }); diff --git a/src/javascript/sourceMaps/SourceMapDecoder.test.ts b/src/javascript/sourceMaps/SourceMapDecoder.test.ts index 9236716b0f..7e08c96d4f 100644 --- a/src/javascript/sourceMaps/SourceMapDecoder.test.ts +++ b/src/javascript/sourceMaps/SourceMapDecoder.test.ts @@ -124,6 +124,7 @@ it("settles every source-map cleanup failure when final close releases the exact prefix: "rea-source-map-final-cleanup-test-", }); let attempts = 0; + let denyCleanup = true; const launchCause = new Error("controlled launch failure"); const decoder = new SourceMapDecoder({ prepareOwnership: async () => {}, @@ -131,7 +132,7 @@ it("settles every source-map cleanup failure when final close releases the exact path: root.path, close: async () => { attempts += 1; - if (attempts <= 2) + if (denyCleanup && attempts <= 2) throw new Error(`runtime cleanup attempt ${String(attempts)} failed`); await root.close(); }, @@ -177,6 +178,7 @@ it("settles every source-map cleanup failure when final close releases the exact "runtime cleanup attempt 2 failed", ]); } finally { + denyCleanup = false; await decoder.close(); } }); @@ -186,6 +188,7 @@ it("attributes cleanup resources to the remaining process and directory owners", prefix: "rea-source-map-mixed-cleanup-test-", }); let denyProcessRelease = true; + let denyRootRelease = true; let rootAttempts = 0; let runId: string | undefined; const decoder = new SourceMapDecoder({ @@ -194,7 +197,7 @@ it("attributes cleanup resources to the remaining process and directory owners", path: root.path, close: async () => { rootAttempts += 1; - if (rootAttempts === 1) + if (denyRootRelease && rootAttempts === 1) throw new Error("controlled directory cleanup failure"); await root.close(); }, @@ -235,6 +238,7 @@ it("attributes cleanup resources to the remaining process and directory owners", await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); } finally { denyProcessRelease = false; + denyRootRelease = false; await decoder.close(); } }); diff --git a/tests/evaluation/model-evals/agentEvaluationReport.test.ts b/tests/evaluation/model-evals/agentEvaluationReport.test.ts index d51c59e7ab..8d2f8375cd 100644 --- a/tests/evaluation/model-evals/agentEvaluationReport.test.ts +++ b/tests/evaluation/model-evals/agentEvaluationReport.test.ts @@ -1,7 +1,5 @@ -import { readFile } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; -import { parse } from "@babel/parser"; -import * as t from "@babel/types"; import { describe, expect, it } from "vitest"; import { @@ -22,102 +20,40 @@ const routingSuccess = { }; describe("agent evaluation release gate", () => { - it("rejects incorrect fixture claims even when every text heuristic passes", async () => { - const predicate = await runnerFailurePredicate(); - expect(predicate({ ...routingSuccess, factualCorrectness: "passed" })).toBe( - false, + it("exits unsuccessfully for a rejected report after accepting a passing report", () => { + const passing = { + id: "passing-fixture", + ...routingSuccess, + factualCorrectness: "passed", + }; + const accepted = runReportGate([passing]); + expect(accepted.status).toBe(0); + const rejected = runReportGate([ + passing, + { ...passing, id: "incorrect-fixture", factualCorrectness: "failed" }, + ]); + // Negative control: a runner that lets the failed report pass exits zero. + expect(rejected.status).toBe(1); + expect(rejected.stderr).toContain( + "Agent routing/workflow/answer checks failed: incorrect-fixture", ); - expect( - predicate({ - exitCode: 0, - naturalUse: true, - correctFirstTool: true, - repeatedCallCount: 0, - inputValidationFailureCount: 0, - requiredToolSubsequenceMet: true, - inputTokens: 100, - answerHeuristicsMet: true, - epistemicCuePresent: true, - factualCorrectness: "failed", - }), - ).toBe(true); - }); - - it("selects the results gate inside cleanup without using an unrelated filter", async () => { - const predicate = await runnerFailurePredicate(` - const failed = unrelated.filter(() => true); - await withVerifierCleanup(async () => { - const failed = results.filter(item => !agentEvaluationPassed(item)); - }); - `); - expect(predicate({ ...routingSuccess, factualCorrectness: "passed" })).toBe( - false, - ); - expect(predicate({ ...routingSuccess, factualCorrectness: "failed" })).toBe( - true, - ); - }); - - it("rejects a missing results failure gate", async () => { - await expect( - runnerFailurePredicate("const failed = unrelated.filter(() => false);"), - ).rejects.toThrow("Expected one runner failure gate, found 0"); - }); - - it("rejects ambiguous results failure gates", async () => { - await expect( - runnerFailurePredicate(` - function first() { const failed = results.filter(item => !agentEvaluationPassed(item)); } - function second() { const failed = results.filter(() => true); } - `), - ).rejects.toThrow("Expected one runner failure gate, found 2"); }); }); -async function runnerFailurePredicate( - providedSource?: string, -): Promise<(item: unknown) => unknown> { - const source = - providedSource ?? - (await readFile("scripts/verify-agent-experience.mjs", "utf8")); - const program = parse(source, { sourceType: "module" }).program; - const predicates: { readonly start: number; readonly end: number }[] = []; - t.traverseFast(program, (node) => { - if ( - !t.isVariableDeclarator(node) || - !t.isIdentifier(node.id, { name: "failed" }) - ) - return; - const initializer = node.init; - if ( - !t.isCallExpression(initializer) || - !t.isMemberExpression(initializer.callee) || - initializer.callee.computed || - !t.isIdentifier(initializer.callee.object, { name: "results" }) || - !t.isIdentifier(initializer.callee.property, { name: "filter" }) - ) - return; - const argument = initializer.arguments[0]; - if ( - t.isArrowFunctionExpression(argument) && - typeof argument.start === "number" && - typeof argument.end === "number" - ) - predicates.push({ start: argument.start, end: argument.end }); - }); - const [predicate] = predicates; - if (predicates.length !== 1 || predicate === undefined) - throw new Error( - `Expected one runner failure gate, found ${predicates.length}`, - ); - const factory = new Function( - "agentEvaluationPassed", - `return (${source.slice(predicate.start, predicate.end)});`, +function runReportGate(results: readonly object[]) { + const gateUrl = new URL( + "../../../scripts/lib/agent-evaluation-report-gate.mjs", + import.meta.url, + ); + const script = ` + const { assertAgentEvaluationReportPassed } = await import(${JSON.stringify(gateUrl.href)}); + assertAgentEvaluationReportPassed(JSON.parse(process.argv[1])); + `; + return spawnSync( + process.execPath, + ["--input-type=module", "-e", script, JSON.stringify(results)], + { encoding: "utf8", timeout: 15_000 }, ); - const result: unknown = factory(agentEvaluationPassed); - if (typeof result !== "function") - throw new Error("Runner gate is not callable"); - return (item) => result(item); } describe("scoped factual report", () => { From fdf5a33fe23e2ad15fba93e80502e5aa6438aa1f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:28:27 +0800 Subject: [PATCH 35/37] refactor(tests): keep fixture adapters outside runtime --- ...ReconstructionObligationCandidates.test.ts | 6 +- src/artifacts/inventory/classify.ts | 31 ------ src/cliCommandNames.ts | 88 ----------------- src/domain/native/peLoadConfig.ts | 6 -- src/domain/process/processCapture.fixture.ts | 33 +++++++ .../process/processCapture.validation.test.ts | 6 +- src/domain/process/processCaptureExample.ts | 28 ------ src/domain/zipPackageFormat.ts | 2 +- src/il2cpp/Il2cppBinaryFile.ts | 21 ----- src/il2cpp/Il2cppMetadataFile.ts | 20 ---- src/native/pe/PeLoadConfig.fixture.ts | 8 +- src/server/EvidenceMcpServer.test.ts | 6 +- src/server/EvidenceMcpServer.ts | 5 +- tests/boundary/cli/cliToolCoverage.test.ts | 94 ++++++++++++++++++- .../boundary/il2cpp/il2cppBinaryFile.test.ts | 14 +-- .../il2cpp/il2cppMappingOwnership.test.ts | 14 ++- .../il2cpp/il2cppMetadataFile.test.ts | 6 +- .../processComparisonDiagnosticsMcp.test.ts | 6 +- .../processFilesystemCoverageAdapters.test.ts | 2 +- tests/fixtures/artifactInventory.ts | 31 +++++- tests/fixtures/il2cpp/readers.ts | 26 +++++ 21 files changed, 206 insertions(+), 247 deletions(-) create mode 100644 tests/fixtures/il2cpp/readers.ts diff --git a/src/application/ReconstructionObligationCandidates.test.ts b/src/application/ReconstructionObligationCandidates.test.ts index 759a060e8f..9051685d8b 100644 --- a/src/application/ReconstructionObligationCandidates.test.ts +++ b/src/application/ReconstructionObligationCandidates.test.ts @@ -13,8 +13,10 @@ import { describe, expect, it } from "vitest"; import { createEvidence } from "../domain/evidence.js"; import { createEvidenceBundle } from "../domain/evidenceBundle.js"; -import { emptyUnverifiedProcessCapture } from "../domain/process/processCapture.fixture.js"; -import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "../domain/process/processCaptureExample.js"; +import { + emptyUnverifiedProcessCapture, + FINALIZED_PROCESS_CAPTURE_EXAMPLE, +} from "../domain/process/processCapture.fixture.js"; import { PROCESS_PROVIDER } from "../domain/process/processEvidenceProvider.js"; import { parseProcessCapture } from "../domain/process/processCaptureParsing.js"; import { diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 6bc93137b3..466ada17b2 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -49,37 +49,6 @@ export interface RootInventorySource extends StableRegularFileDescriptor { readonly owner: OwnedFileHandle; } -/** Classify and hash one file root through the same stable open descriptor. */ -export const classifyAndHashRoot = async ( - path: string, - directory: boolean, - expectedMetadata: Stats, - options: Pick, -): Promise => { - return options.resourceScope.run(async () => { - const result = await classifyAndHashRootForInventory( - path, - directory, - expectedMetadata, - options, - ); - const rootSource = result.rootSource; - if (rootSource !== undefined) { - const owner: ArtifactResourceOwner = { - kind: "file-handle", - handle: rootSource.owner, - resource: `root artifact descriptor for ${path}`, - }; - const cleanupAttempt = await options.resourceScope.release(owner); - if (cleanupAttempt.kind === "failed") - throw options.resourceScope.cleanupFailure({ kind: "completed" }, [ - cleanupAttempt, - ]); - } - return { format: result.format, digest: result.digest }; - }); -}; - /** Classify a root and retain its admitted ZIP descriptor for child inventory. */ export const classifyAndHashRootForInventory = async ( path: string, diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index f1e8cf0132..b56abae118 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -180,91 +180,3 @@ export const CLI_COMMANDS = Object.freeze({ /** Ordered primary CLI inventory; aliases are intentionally excluded. */ export const CLI_COMMAND_NAMES = Object.freeze(Object.values(CLI_COMMANDS)); - -/** Canonical MCP operations reached by CLI commands with different names. */ -export const CLI_COMMAND_TOOL_ALIASES = Object.freeze({ - "environment-build": ["export_environment_bundle"], - analyze: ["binary_overview"], - inspect: ["binary_overview"], - decompile: ["procedure_pseudo_code"], - function: ["analyze_function"], - instructions: ["read_function_instructions"], - trace: ["trace_feature"], - search: ["search_strings", "search_procedures"], - "capture-process": ["capture_process_scenario"], - "compare-bundles": ["compare_bundles"], - "list-objdump-architectures": ["inspect_with_objdump"], - "debug-with-gdb": ["gdb_console"], - "debug-with-rizin": ["rizin_debug_command"], -}); - -/** MCP operations without a dedicated CLI command or direct command alias. */ -export const MCP_TOOLS_WITHOUT_DEDICATED_CLI = Object.freeze([ - // MCP session control selects a resource root; CLI resource commands take paths directly. - "open_bundle_resources", - "address_name", - "comment", - "current_address", - "current_procedure", - "current_document", - "goto_address", - "inline_comment", - "list_bookmarks", - "list_documents", - "list_names", - "list_procedures", - "list_segments", - "list_strings", - "next_address", - "prev_address", - "procedure_address", - "procedure_assembly", - "procedure_callees", - "procedure_callers", - "procedure_info", - "procedure_references", - "resolve_containing_procedure", - "set_address_name", - "set_addresses_names", - "set_bookmark", - "set_comment", - "set_inline_comment", - "unset_bookmark", - "get_objc_classes", - "get_objc_protocols", - "batch_decompile", - "get_call_graph", - "analyze_swift_types", - "find_xrefs_to_name", - "trace_call_path", - "open_binary", - "close_binary", - "binary_session", - "export_evidence_bundle", - "import_evidence_bundle", - "compare_artifacts", - "compare_functions", - "find_changed_behavior", - "build_call_path", - "correlate_static_and_runtime", - "verify_reconstruction", - "list_unknowns", - "record_unknown", - "update_unknown", - "verify_unknown_resolution", - "get_evidence_bundle", - "get_navigation_context", - "inspect_address_context", - "start_gdb_session", - "gdb_session_status", - "close_gdb_session", - "start_rizin_debug_session", - "rizin_debug_session_status", - "close_rizin_debug_session", - "start_frida_session", - "load_frida_script", - "resume_frida_session", - "unload_frida_script", - "frida_session_status", - "close_frida_session", -]); diff --git a/src/domain/native/peLoadConfig.ts b/src/domain/native/peLoadConfig.ts index f6a23fb750..7b3f8ff07c 100644 --- a/src/domain/native/peLoadConfig.ts +++ b/src/domain/native/peLoadConfig.ts @@ -16,12 +16,6 @@ export const MAX_INSPECT_PE_LOAD_CONFIG_CALLBACKS = 4096; /** IMAGE_TLS_DIRECTORY byte width for each optional-header magic. */ export const TLS_STRUCT_BYTES = { 32: 24, 64: 40 } as const; -/** - * IMAGE_LOAD_CONFIG_DIRECTORY size through GuardMemcpyFunctionPointer. - * Later fields stay unread; a larger Size does not fail the inspection. - */ -export const LOAD_CONFIG_KNOWN_STRUCT_BYTES = { 32: 192, 64: 320 } as const; - /** Complete bounded inspection of one PE TLS directory and load configuration. */ export const inspectPeLoadConfigInputSchema = z.strictObject({ path: absoluteLocalPathStringSchema.describe( diff --git a/src/domain/process/processCapture.fixture.ts b/src/domain/process/processCapture.fixture.ts index a8f2e669f8..abe269f968 100644 --- a/src/domain/process/processCapture.fixture.ts +++ b/src/domain/process/processCapture.fixture.ts @@ -3,11 +3,44 @@ import { compareProcessCaptures } from "./processComparison.js"; import { parseProcessCapture } from "./processCaptureParsing.js"; import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "./processCaptureExample.js"; import { hasCaptureTruncation } from "./processCaptureCoverage.js"; +import { digestProcessCommitment } from "./processScenario.js"; +import type { JsonValue } from "../jsonValue.js"; import { processCaptureSchema, type UnverifiedProcessCapture, } from "./processCapture.js"; + +const finalizedScenario = { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.scenario, + finalization_ms: 1_500, +}; +const finalizedComparisonContract = { finalization_ms: 1_500 }; + +/** Valid capture of a timeout that finalized: SIGTERM delivered, exit observed. */ +export const FINALIZED_PROCESS_CAPTURE_EXAMPLE = { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE, + manifest: { + ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest, + scenario: finalizedScenario, + comparison_contract: finalizedComparisonContract, + full_scenario_sha256: digestProcessCommitment(finalizedScenario), + comparison_contract_sha256: digestProcessCommitment( + finalizedComparisonContract, + ), + }, + exit: { + code: null, + signal: 0, + reason: "timeout", + finalization: { + requested_ms: 1_500, + elapsed_ms: 12, + exit_code: 0, + signals: [{ signal: "SIGTERM", sent_at_ms: 0, delivery: "signaled" }], + }, + }, +} satisfies JsonValue; /** Parse a detached valid empty capture for tests that need trusted evidence. */ export const emptyProcessCapture = () => parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE); diff --git a/src/domain/process/processCapture.validation.test.ts b/src/domain/process/processCapture.validation.test.ts index e0f2f72e75..325cfe37ed 100644 --- a/src/domain/process/processCapture.validation.test.ts +++ b/src/domain/process/processCapture.validation.test.ts @@ -8,14 +8,14 @@ import { } from "./processCaptureParsing.js"; import type { UnverifiedProcessCapture } from "./processCapture.js"; import { partialProcessCaptureObservationSchema } from "./processCapture.js"; -import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "./processCaptureExample.js"; -import { finalizationConsistencyIssue } from "./processCaptureValidation.js"; -import type { FilesystemCoverage } from "./processCaptureCoverage.js"; import { + FINALIZED_PROCESS_CAPTURE_EXAMPLE, compareUnverifiedProcessCaptures as compareProcessCaptures, emptyUnverifiedProcessCapture as emptyCapture, processCaptureIssues, } from "./processCapture.fixture.js"; +import { finalizationConsistencyIssue } from "./processCaptureValidation.js"; +import type { FilesystemCoverage } from "./processCaptureCoverage.js"; expectTypeOf().not.toExtend(); diff --git a/src/domain/process/processCaptureExample.ts b/src/domain/process/processCaptureExample.ts index 8259f38501..0b01a0e03f 100644 --- a/src/domain/process/processCaptureExample.ts +++ b/src/domain/process/processCaptureExample.ts @@ -111,31 +111,3 @@ export const EMPTY_PROCESS_CAPTURE_EXAMPLE = { temporary_root: "removed", }, } satisfies JsonValue; - -const finalizedScenario = { ...scenario, finalization_ms: 1_500 }; -const finalizedComparisonContract = { finalization_ms: 1_500 }; - -/** Valid capture of a timeout that finalized: SIGTERM delivered, exit observed. */ -export const FINALIZED_PROCESS_CAPTURE_EXAMPLE = { - ...EMPTY_PROCESS_CAPTURE_EXAMPLE, - manifest: { - ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest, - scenario: finalizedScenario, - comparison_contract: finalizedComparisonContract, - full_scenario_sha256: digestProcessCommitment(finalizedScenario), - comparison_contract_sha256: digestProcessCommitment( - finalizedComparisonContract, - ), - }, - exit: { - code: null, - signal: 0, - reason: "timeout", - finalization: { - requested_ms: 1_500, - elapsed_ms: 12, - exit_code: 0, - signals: [{ signal: "SIGTERM", sent_at_ms: 0, delivery: "signaled" }], - }, - }, -} satisfies JsonValue; diff --git a/src/domain/zipPackageFormat.ts b/src/domain/zipPackageFormat.ts index 1601166804..93bd1b9461 100644 --- a/src/domain/zipPackageFormat.ts +++ b/src/domain/zipPackageFormat.ts @@ -105,7 +105,7 @@ export const locateTrailingZipArchive = async ( * is intentionally absent: the suffix also names HTTP Archive JSON, so a * Harmony shared archive is only established from verified ZIP bytes. The * HarmonyOS App Pack suffix `.app` is also absent here: it applies only to - * root inputs (see `classifyAndHashRoot`), because a nested ZIP entry ending + * root inputs (see `classifyAndHashRootForInventory`), because a nested ZIP entry ending * in `.app` inside another package, such as an IPA, is not an App Pack. */ export const zipPackageFormatForPath = ( diff --git a/src/il2cpp/Il2cppBinaryFile.ts b/src/il2cpp/Il2cppBinaryFile.ts index 0bcd18d74c..9dc93f4e3a 100644 --- a/src/il2cpp/Il2cppBinaryFile.ts +++ b/src/il2cpp/Il2cppBinaryFile.ts @@ -268,24 +268,3 @@ function* referencedTypeCount( /** Rows between cooperative yields while linking. */ const LINK_BATCH = 256; - -/** Synchronous linking for byte-level callers. */ -export const linkIl2cppBinary = ( - bytes: Buffer, - metadata: Il2cppMetadataDecoded, - facts: Il2cppMetadataLinkFacts, - architecture?: string, - options: Il2cppBinaryReadOptions = {}, -): Il2cppBinaryLink => { - const steps = linkIl2cppBinarySteps( - bytes, - metadata, - facts, - architecture, - options, - ); - for (;;) { - const step = steps.next(); - if (step.done) return step.value; - } -}; diff --git a/src/il2cpp/Il2cppMetadataFile.ts b/src/il2cpp/Il2cppMetadataFile.ts index d98d317b0a..b20ce0e81f 100644 --- a/src/il2cpp/Il2cppMetadataFile.ts +++ b/src/il2cpp/Il2cppMetadataFile.ts @@ -969,23 +969,3 @@ function* readMetadataCoreSteps( link: tables.linkFacts, }; } - -const runSteps = (steps: Generator): T => { - for (;;) { - const step = steps.next(); - if (step.done) return step.value; - } -}; - -/** Synchronous consumption of the same decoder for byte-level callers. */ -export const readIl2cppMetadata = ( - bytes: Buffer, - options: Il2cppMetadataReadOptions = {}, -): Il2cppMetadataDecoded => runSteps(readIl2cppMetadataSteps(bytes, options)); - -/** Synchronous decoding that also retains the typed row facts used for native linking. */ -export const readIl2cppMetadataForLinking = ( - bytes: Buffer, - options: Il2cppMetadataReadOptions = {}, -): Il2cppMetadataForLinking => - runSteps(readIl2cppMetadataForLinkingSteps(bytes, options)); diff --git a/src/native/pe/PeLoadConfig.fixture.ts b/src/native/pe/PeLoadConfig.fixture.ts index ed3cdb6f52..a0f447c79e 100644 --- a/src/native/pe/PeLoadConfig.fixture.ts +++ b/src/native/pe/PeLoadConfig.fixture.ts @@ -1,7 +1,7 @@ -import { - LOAD_CONFIG_KNOWN_STRUCT_BYTES, - loadConfigFieldLayout, -} from "../../domain/native/peLoadConfig.js"; +import { loadConfigFieldLayout } from "../../domain/native/peLoadConfig.js"; + +/** Authored fixture directory sizes through GuardMemcpyFunctionPointer. */ +const LOAD_CONFIG_KNOWN_STRUCT_BYTES = { 32: 192, 64: 320 } as const; /** File and RVA layout shared by the synthetic PE32 and PE32+ images. */ export const PE_LOAD_CONFIG_FIXTURE = { diff --git a/src/server/EvidenceMcpServer.test.ts b/src/server/EvidenceMcpServer.test.ts index f55ab9e450..a5b69e279e 100644 --- a/src/server/EvidenceMcpServer.test.ts +++ b/src/server/EvidenceMcpServer.test.ts @@ -13,10 +13,8 @@ import { serveStdio } from "@modelcontextprotocol/server/stdio"; import { z } from "zod"; import { InvestigationRecords } from "../application/investigation/InvestigationRecords.js"; -import { - EvidenceMcpServer, - forgetPendingMcpRequest, -} from "./EvidenceMcpServer.js"; +import { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import { forgetPendingMcpRequest } from "./EvidenceMcpServerCore.js"; import { createToolResultDelivery } from "./toolResult.js"; const firstText = (result: { diff --git a/src/server/EvidenceMcpServer.ts b/src/server/EvidenceMcpServer.ts index 5d18cf0501..bd7c78331d 100644 --- a/src/server/EvidenceMcpServer.ts +++ b/src/server/EvidenceMcpServer.ts @@ -13,10 +13,7 @@ import { type CompactInputSchemaPresentation, } from "./EvidenceMcpServerCore.js"; -export { - forgetPendingMcpRequest, - type CompactInputSchemaPresentation, -} from "./EvidenceMcpServerCore.js"; +export type { CompactInputSchemaPresentation } from "./EvidenceMcpServerCore.js"; /** Canonical composition retains the constructor's existing identity defaults. */ export class EvidenceMcpServer extends EvidenceMcpServerCore { diff --git a/tests/boundary/cli/cliToolCoverage.test.ts b/tests/boundary/cli/cliToolCoverage.test.ts index 5067e001ce..8b4f71346d 100644 --- a/tests/boundary/cli/cliToolCoverage.test.ts +++ b/tests/boundary/cli/cliToolCoverage.test.ts @@ -3,11 +3,95 @@ import { describe, expect, it } from "vitest"; import { createCli } from "../../../src/cli.js"; import { createCliInventory } from "../../../scripts/lib/catalog-cli.mjs"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; -import { - CLI_COMMAND_NAMES, - CLI_COMMAND_TOOL_ALIASES, - MCP_TOOLS_WITHOUT_DEDICATED_CLI, -} from "../../../src/cliCommandNames.js"; +import { CLI_COMMAND_NAMES } from "../../../src/cliCommandNames.js"; + +/** Canonical MCP operations reached by CLI commands with different names. */ +const CLI_COMMAND_TOOL_ALIASES = Object.freeze({ + "environment-build": ["export_environment_bundle"], + analyze: ["binary_overview"], + inspect: ["binary_overview"], + decompile: ["procedure_pseudo_code"], + function: ["analyze_function"], + instructions: ["read_function_instructions"], + trace: ["trace_feature"], + search: ["search_strings", "search_procedures"], + "capture-process": ["capture_process_scenario"], + "compare-bundles": ["compare_bundles"], + "list-objdump-architectures": ["inspect_with_objdump"], + "debug-with-gdb": ["gdb_console"], + "debug-with-rizin": ["rizin_debug_command"], +}); + +/** MCP operations without a dedicated CLI command or direct command alias. */ +const MCP_TOOLS_WITHOUT_DEDICATED_CLI = Object.freeze([ + // MCP session control selects a resource root; CLI resource commands take paths directly. + "open_bundle_resources", + "address_name", + "comment", + "current_address", + "current_procedure", + "current_document", + "goto_address", + "inline_comment", + "list_bookmarks", + "list_documents", + "list_names", + "list_procedures", + "list_segments", + "list_strings", + "next_address", + "prev_address", + "procedure_address", + "procedure_assembly", + "procedure_callees", + "procedure_callers", + "procedure_info", + "procedure_references", + "resolve_containing_procedure", + "set_address_name", + "set_addresses_names", + "set_bookmark", + "set_comment", + "set_inline_comment", + "unset_bookmark", + "get_objc_classes", + "get_objc_protocols", + "batch_decompile", + "get_call_graph", + "analyze_swift_types", + "find_xrefs_to_name", + "trace_call_path", + "open_binary", + "close_binary", + "binary_session", + "export_evidence_bundle", + "import_evidence_bundle", + "compare_artifacts", + "compare_functions", + "find_changed_behavior", + "build_call_path", + "correlate_static_and_runtime", + "verify_reconstruction", + "list_unknowns", + "record_unknown", + "update_unknown", + "verify_unknown_resolution", + "get_evidence_bundle", + "get_navigation_context", + "inspect_address_context", + "start_gdb_session", + "gdb_session_status", + "close_gdb_session", + "start_rizin_debug_session", + "rizin_debug_session_status", + "close_rizin_debug_session", + "start_frida_session", + "load_frida_script", + "resume_frida_session", + "unload_frida_script", + "frida_session_status", + "close_frida_session", +]); describe("CLI canonical tool coverage", () => { it("accounts for every MCP tool on the CLI or marks it intentionally MCP-only", () => { diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index cf918e9812..830de01553 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -5,11 +5,12 @@ import { readUInt32, } from "../../../src/il2cpp/Il2cppBinaryImage.js"; import { expect, it } from "vitest"; +import { linkIl2cppBinarySteps } from "../../../src/il2cpp/Il2cppBinaryFile.js"; import { + drain, linkIl2cppBinary, - linkIl2cppBinarySteps, -} from "../../../src/il2cpp/Il2cppBinaryFile.js"; -import { readIl2cppMetadataForLinking } from "../../../src/il2cpp/Il2cppMetadataFile.js"; + readIl2cppMetadataForLinking, +} from "../../fixtures/il2cpp/readers.js"; import { GENERIC_TABLE_LIMIT, linkGenericMethodsSteps, @@ -1673,13 +1674,6 @@ it("returns no instances for an empty genericMethodTable without picking a layou }); }); -const drain = (steps: Generator, onYield = () => {}): T => { - for (;;) { - const step = steps.next(); - if (step.done === true) return step.value; - onYield(); - } -}; // Link a low-base image of generic tables only, recording yields and rows. const linkLowBase = ( options: GenericRegistrationImageOptions = {}, diff --git a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts index 908b835a53..a166ca7dfa 100644 --- a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts +++ b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts @@ -1,5 +1,4 @@ import { expect, it, vi } from "vitest"; -import { linkIl2cppBinary } from "../../../src/il2cpp/Il2cppBinaryFile.js"; import { isMapped, openImageSteps, @@ -7,7 +6,11 @@ import { readSlot, readUInt32, } from "../../../src/il2cpp/Il2cppBinaryImage.js"; -import { readIl2cppMetadataForLinking } from "../../../src/il2cpp/Il2cppMetadataFile.js"; +import { + drain, + linkIl2cppBinary, + readIl2cppMetadataForLinking, +} from "../../fixtures/il2cpp/readers.js"; import { createElfFixture, createIl2cppBinaryFixture, @@ -20,12 +23,7 @@ const { metadata, link: facts } = readIl2cppMetadataForLinking( createLinkedIl2cppMetadataFixture().bytes, ); const link = (bytes: Buffer) => linkIl2cppBinary(bytes, metadata, facts); -const open = (bytes: Buffer) => { - const steps = openImageSteps(bytes, undefined); - let step = steps.next(); - while (!step.done) step = steps.next(); - return step.value; -}; +const open = (bytes: Buffer) => drain(openImageSteps(bytes, undefined)); /** Replace PT_LOAD declarations, keeping the original PT_DYNAMIC and payload. */ const withLoadRanges = ( diff --git a/tests/boundary/il2cpp/il2cppMetadataFile.test.ts b/tests/boundary/il2cpp/il2cppMetadataFile.test.ts index 6fdc381cd0..41770d777b 100644 --- a/tests/boundary/il2cpp/il2cppMetadataFile.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataFile.test.ts @@ -6,10 +6,12 @@ import { describe, expect, it } from "vitest"; import { il2cppIndexWidth, il2cppMetadataRecordBytes, - readIl2cppMetadata, - readIl2cppMetadataForLinking, readIl2cppMetadataSteps, } from "../../../src/il2cpp/Il2cppMetadataFile.js"; +import { + readIl2cppMetadata, + readIl2cppMetadataForLinking, +} from "../../fixtures/il2cpp/readers.js"; import { createIl2cppMetadataFixture, FIXTURE_IMAGES, diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts index b4fe9c8fdf..d9554cd390 100644 --- a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts +++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts @@ -7,10 +7,8 @@ import { expect, it, onTestFinished } from "vitest"; import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js"; import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; -import { - EMPTY_PROCESS_CAPTURE_EXAMPLE, - FINALIZED_PROCESS_CAPTURE_EXAMPLE, -} from "../../../src/domain/process/processCaptureExample.js"; +import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; +import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCapture.fixture.js"; import { capture, partialSpecification, diff --git a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts index 086b38ca8e..1772af643c 100644 --- a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts +++ b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts @@ -17,7 +17,7 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { compareProcessCaptures } from "../../../src/domain/process/processComparison.js"; import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; import { processScenarioSchema } from "../../../src/domain/process/processScenario.js"; -import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; +import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCapture.fixture.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; diff --git a/tests/fixtures/artifactInventory.ts b/tests/fixtures/artifactInventory.ts index 948289a415..709946bebd 100644 --- a/tests/fixtures/artifactInventory.ts +++ b/tests/fixtures/artifactInventory.ts @@ -1,7 +1,10 @@ -import { ArtifactResourceScope } from "../../src/artifacts/ArtifactResourceScope.js"; +import { + ArtifactResourceScope, + type ArtifactResourceOwner, +} from "../../src/artifacts/ArtifactResourceScope.js"; import { onTestFinished } from "vitest"; import { ArtifactReaderFailure } from "../../src/artifacts/ArtifactReader.js"; -import { classifyAndHashRoot as classifyAndHashRootOwned } from "../../src/artifacts/inventory/classify.js"; +import { classifyAndHashRootForInventory } from "../../src/artifacts/inventory/classify.js"; import { inventoryArtifact as inventoryArtifactOwned, scanArtifactInventoryInScope, @@ -32,9 +35,27 @@ export const classifyAndHashRoot = ( signal?: AbortSignal, ) => withArtifactResourceScope((resourceScope) => - classifyAndHashRootOwned(path, directory, expectedMetadata, { - resourceScope, - ...(signal === undefined ? {} : { signal }), + resourceScope.run(async () => { + const result = await classifyAndHashRootForInventory( + path, + directory, + expectedMetadata, + { resourceScope, ...(signal === undefined ? {} : { signal }) }, + ); + const rootSource = result.rootSource; + if (rootSource !== undefined) { + const owner: ArtifactResourceOwner = { + kind: "file-handle", + handle: rootSource.owner, + resource: `root artifact descriptor for ${path}`, + }; + const cleanupAttempt = await resourceScope.release(owner); + if (cleanupAttempt.kind === "failed") + throw resourceScope.cleanupFailure({ kind: "completed" }, [ + cleanupAttempt, + ]); + } + return { format: result.format, digest: result.digest }; }), ); diff --git a/tests/fixtures/il2cpp/readers.ts b/tests/fixtures/il2cpp/readers.ts new file mode 100644 index 0000000000..5ee91b467f --- /dev/null +++ b/tests/fixtures/il2cpp/readers.ts @@ -0,0 +1,26 @@ +import { linkIl2cppBinarySteps } from "../../../src/il2cpp/Il2cppBinaryFile.js"; +import { + readIl2cppMetadataForLinkingSteps, + readIl2cppMetadataSteps, +} from "../../../src/il2cpp/Il2cppMetadataFile.js"; + +/** Consume production reader steps synchronously, optionally observing each yield. */ +export const drain = (steps: Generator, onYield = () => {}): T => { + for (;;) { + const step = steps.next(); + if (step.done === true) return step.value; + onYield(); + } +}; + +export const readIl2cppMetadata = ( + ...parameters: Parameters +) => drain(readIl2cppMetadataSteps(...parameters)); + +export const readIl2cppMetadataForLinking = ( + ...parameters: Parameters +) => drain(readIl2cppMetadataForLinkingSteps(...parameters)); + +export const linkIl2cppBinary = ( + ...parameters: Parameters +) => drain(linkIl2cppBinarySteps(...parameters)); From 5938581b3733e955d8efc21d38b36be5ba08e15d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:28:52 +0800 Subject: [PATCH 36/37] fix(async): own completion across test and provider boundaries --- scripts/fixtures/browser-network-site.mjs | 12 +- scripts/fixtures/browser-request-site.mjs | 14 +- scripts/lib/verifier-http-server.d.mts | 12 + scripts/lib/verifier-http-server.mjs | 39 +++ src/application/AppBundleExecutable.ts | 8 +- src/application/LinuxHopperPublisher.ts | 2 +- src/application/SetupClientConfiguration.ts | 4 +- .../environment/environmentExport.ts | 2 +- .../pyinstaller/PyInstallerBoundaries.test.ts | 8 +- src/browser/PlaywrightScenarioSession.test.ts | 37 ++- src/gdb/GdbSessionManager.ts | 24 +- src/ghidra/GhidraClient.ts | 2 +- src/hopper/HopperClient.ts | 2 +- src/kuna/KunaTargetCommands.ts | 2 +- src/rizin/RizinDebugSessionManager.ts | 15 +- .../boundary/cli/cliWorkerCompletion.test.ts | 160 ++++++++++ .../cli/idaStartupAttributionCli.test.ts | 274 ++++++++++-------- .../browserFixtureRequestFailures.test.ts | 67 +++++ .../filesystem/ghidraPspExtension.test.ts | 5 +- .../referenceSourceCleanupOwnership.test.ts | 33 ++- tests/boundary/mcp/idaTransport.test.ts | 173 +++++------ .../process/domCallbackFailures.test.ts | 210 ++++++++++++++ .../process/fixtureAsyncFailures.test.ts | 78 +++++ .../process/websiteInitialization.test.ts | 47 +++ .../boundary/providers/jebMcpCatalog.test.ts | 86 +++--- .../readiness/electron/renderer.js | 9 +- tests/fixtures/evidencePublicationPreload.mjs | 44 +-- tests/fixtures/fakeGhidra.mjs | 14 +- tests/fixtures/hopperLaunchAdmission.mjs | 24 +- tests/fixtures/processCaptureCancellation.mjs | 41 ++- .../filesystemSnapshotRaces.test.ts | 5 +- .../mcpDeliveryEnvironment.test.ts | 4 +- .../referenceSourceSymlinkRace.test.ts | 5 +- tests/support/cli/cliCommandSessionWorker.mjs | 56 +++- website/public/assets/main.js | 6 +- website/public/assets/theme.js | 7 +- .../examples/notes-electron/renderer.js | 9 +- 37 files changed, 1134 insertions(+), 406 deletions(-) create mode 100644 scripts/lib/verifier-http-server.d.mts create mode 100644 scripts/lib/verifier-http-server.mjs create mode 100644 tests/boundary/cli/cliWorkerCompletion.test.ts create mode 100644 tests/boundary/filesystem/browserFixtureRequestFailures.test.ts create mode 100644 tests/boundary/process/domCallbackFailures.test.ts create mode 100644 tests/boundary/process/fixtureAsyncFailures.test.ts create mode 100644 tests/boundary/process/websiteInitialization.test.ts diff --git a/scripts/fixtures/browser-network-site.mjs b/scripts/fixtures/browser-network-site.mjs index 6973ce4055..7e8cc64b50 100644 --- a/scripts/fixtures/browser-network-site.mjs +++ b/scripts/fixtures/browser-network-site.mjs @@ -1,5 +1,5 @@ +import { createVerifierHttpServer } from "../lib/verifier-http-server.mjs"; import { once } from "node:events"; -import { createServer } from "node:http"; import { gzipSync } from "node:zlib"; export const networkFixtureSecret = "私密-network-fixture"; @@ -23,7 +23,7 @@ const html = `