diff --git a/docs/development/testing.md b/docs/development/testing.md index b75654852b..308012aba0 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -48,6 +48,20 @@ that invoke PowerShell themselves remain subject to their own policy checks. | `npm run docs:check` | Generated-document validation and fresh onboarding example consistency checks | | `npm run docs:build` | Render VitePress documentation and apply its page-link checks; no live external-link or GitHub fragment verification | +`typecheck` also checks the JavaScript verification scripts listed in +`tsconfig.scripts.json` (`checkJs`, with the main compiler options). Those +scripts import built modules through the package-private `#dist/*` alias, which +Node resolves to `./dist/*` at run time and the check resolves to `./src/*`, so a +domain signature change that breaks a listed script fails `check:fast` without a +build. To add a script, switch its `../dist/` imports and those of the local +modules it loads to `#dist/`, add it to the list, and fix what the check +reports with JSDoc types. Name tool results with the +`scripts/lib/mcp-verifier-results.mjs` typedefs (`ToolName`, +`VerifierOperationResult`) so a result reads with its named contract's +type. `tests/support/analysis-efficiency/android-mapping-retention.mjs` stays +outside the check: it imports `dist` modules by path from a caller-selected +checkout at run time, so no import in this tree names their types. + The general source suite above runs on Linux in CI, and Windows CI runs a curated lane ([Native platform baseline in CI](#native-platform-baseline-in-ci)). A Windows source checkout can fail source tests for reasons unrelated to the diff --git a/package.json b/package.json index 35c6aa244c..4d791bacce 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ }, "type": "module", "imports": { + "#dist/*": "./dist/*", "#javascript-analysis-worker": "./dist/composition/javascriptAnalysisWorkerProcess.js", "#web-bundle-analysis-worker": "./dist/composition/webBundleAnalysisWorkerProcess.js" }, @@ -192,7 +193,7 @@ "test:watch": "vitest --project domain --project services --project adapters --project composition --project conformance --project evaluation --changed", "pretest:watch:all": "npm run test:prepare", "test:watch:all": "vitest --changed", - "typecheck": "node scripts/run-typescript.mjs --noEmit", + "typecheck": "node scripts/run-typescript.mjs --noEmit && node scripts/run-typescript.mjs --noEmit -p tsconfig.scripts.json", "verify:test-temp-hygiene": "npm run test:prepare && node scripts/verify-test-temp-hygiene.mjs", "verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs", "verify:hopper:callers": "npm run build:cached && node scripts/verify-real-hopper-callers.mjs", diff --git a/scripts/lib/mcp-verifier-results.mjs b/scripts/lib/mcp-verifier-results.mjs index 3b3dd89a64..4c49b3d086 100644 --- a/scripts/lib/mcp-verifier-results.mjs +++ b/scripts/lib/mcp-verifier-results.mjs @@ -1,34 +1,98 @@ import { isDeepStrictEqual } from "node:util"; -import { authenticateParsedEvidenceBundle } from "../../dist/domain/evidenceBundle.js"; -import { analysisErrorProjectionSchema } from "../../dist/domain/analysisErrorProjectionSchema.js"; -import { toolContract } from "../../dist/contracts/toolContracts.js"; +import { z } from "zod"; +import { authenticateParsedEvidenceBundle } from "#dist/domain/evidenceBundle.js"; +import { analysisErrorProjectionSchema } from "#dist/domain/analysisErrorProjectionSchema.js"; +import { toolContract } from "#dist/contracts/toolContracts.js"; import { expandProviderCapabilities, expandToolAvailability, providerCapabilitySet, -} from "../../dist/contracts/toolOutputSchemaPrimitives.js"; -import { compareUnicodeCodePoints } from "../../dist/domain/unicodeCodePointOrder.js"; +} from "#dist/contracts/toolOutputSchemaPrimitives.js"; +import { compareUnicodeCodePoints } from "#dist/domain/unicodeCodePointOrder.js"; -/** Read the textual JSON projection from one MCP tool call. */ +/** @typedef {import("@modelcontextprotocol/client").CallToolResult} CallToolResult */ +/** @typedef {(typeof import("#dist/contracts/toolContracts.js").TOOL_CONTRACTS)[number]} PublicToolContract */ +/** @typedef {PublicToolContract["name"]} ToolName */ +/** + * @template {ToolName} N + * @typedef {Extract} NamedToolContract + */ +/** + * @template {PublicToolContract} C + * @typedef {z.output} ToolOutput + */ +/** + * The member an MCP verifier projects: Evidence `normalized_result` or + * lifecycle `result`. + * @template {PublicToolContract} C + * @typedef {ToolOutput extends { readonly normalized_result: infer R } + * ? R + * : ToolOutput extends { readonly result: infer R } + * ? R + * : never} OperationResult + */ +/** @typedef {OperationResult>} SessionStatus */ +/** + * Session status as verifiers read it, with compact capability and tool + * groups expanded. + * @typedef {ReturnType} ExpandedSessionStatus + */ +/** + * @template {ToolName} N + * @typedef {N extends "binary_session" + * ? ExpandedSessionStatus + * : OperationResult>} VerifierOperationResult + */ + +/** + * Read the textual JSON projection from one MCP tool call. + * @param {CallToolResult} result + * @returns {string} + */ export const mcpTextValue = (result) => { - const text = result.content?.find((item) => item.type === "text")?.text; + const text = result.content.find((item) => item.type === "text")?.text; if (typeof text !== "string") throw new Error("Tool result omitted text"); return text; }; +/** + * @param {CallToolResult} result + * @returns {unknown} + */ const jsonValue = (result) => JSON.parse(mcpTextValue(result)); -/** Sort canonical contract names using REA's code-point ordering. */ +/** + * Sort canonical contract names using REA's code-point ordering. + * @param {Iterable} names + * @returns {string[]} + */ export const sortVerifierNames = (names) => [...names].sort(compareUnicodeCodePoints); -/** Read the complete canonical diagnostic from an MCP tool failure. */ +const toolErrorEnvelope = z.object({ error: z.unknown() }); + +/** + * Read the complete canonical diagnostic from an MCP tool failure. + * @param {CallToolResult} result + */ export const requireMcpToolError = (result) => { if (result.isError !== true) throw new Error("Expected an MCP tool failure"); - return analysisErrorProjectionSchema.parse(jsonValue(result).error); + return analysisErrorProjectionSchema.parse( + toolErrorEnvelope.parse(jsonValue(result)).error, + ); }; -/** Validate the complete output against its exact named contract, retaining inline Evidence. */ +/** + * Validate the complete output against its exact named contract, retaining + * inline Evidence. Evidence outputs return the unprojected JSON value that + * satisfied the contract, so downstream parseEvidence authenticates source + * facts rather than a schema projection. + * @template {ToolName} N + * @param {CallToolResult} result + * @param {N} operation + * @param {NamedToolContract} [contract] + * @returns {ToolOutput>} + */ export const requireMcpOutput = ( result, operation, @@ -39,23 +103,32 @@ export const requireMcpOutput = ( if (contract.name !== operation) throw new Error(`${operation} received the ${contract.name} contract`); const text = jsonValue(result); - const output = contract.outputSchema.parse(text); + // parse on a contract union widens to every contract's output; the guard + // above bound this contract to the operation's name. + const output = /** @type {ToolOutput>} */ ( + contract.outputSchema.parse(text) + ); if (result.structuredContent !== undefined) { // The producer emits both representations from the same JSON value. // Compare before schema projection can discard disagreeing source fields. if (!isDeepStrictEqual(text, result.structuredContent)) throw new Error(`${operation} text and structured results disagree`); } - // Downstream parseEvidence must authenticate source facts, not a schema projection. - return Object.hasOwn(output, "normalized_result") ? text : output; + return Object.hasOwn(output, "normalized_result") + ? /** @type {ToolOutput>} */ (text) + : output; }; +/** + * @param {PublicToolContract} contract + * @returns {"normalized_result" | "result"} + */ const operationResultKey = (contract) => { - const shape = contract.outputSchema.shape; - if (shape === undefined) + const schema = contract.outputSchema; + if (!("shape" in schema)) throw new Error(`${contract.name} has an unsupported MCP output schema`); - const evidence = Object.hasOwn(shape, "normalized_result"); - const lifecycle = Object.hasOwn(shape, "result"); + const evidence = Object.hasOwn(schema.shape, "normalized_result"); + const lifecycle = Object.hasOwn(schema.shape, "result"); if (evidence === lifecycle) throw new Error(`${contract.name} has an unsupported MCP output schema`); return evidence ? "normalized_result" : "result"; @@ -66,6 +139,7 @@ const operationResultKey = (contract) => { * has validated the status against the binary_session contract, so a summary * carries summary candidates without capability sets; complete detail must * still expand every candidate's capabilities. + * @param {SessionStatus} status */ export const expandMcpSessionStatus = (status) => ({ ...status, @@ -77,7 +151,11 @@ export const expandMcpSessionStatus = (status) => ({ tool_availability: expandToolAvailability(status.tool_availability), }); -/** Expand provider capability groups while retaining candidate identity/state. */ +/** + * Expand provider capability groups while retaining candidate identity/state. + * @template {{ readonly capabilities: unknown }} Candidate + * @param {readonly Candidate[]} candidates + */ const expandProviderCandidates = (candidates) => candidates.map((candidate) => ({ ...candidate, @@ -86,7 +164,14 @@ const expandProviderCandidates = (candidates) => ), })); -/** Expand CLI provider capability groups at the same owning boundary. */ +/** + * Expand CLI provider capability groups at the same owning boundary. + * @template {{ + * readonly capabilities: unknown; + * readonly analysis_provider_candidates: readonly { readonly capabilities: unknown }[]; + * }} Providers + * @param {Providers} providers + */ export const expandProviderOutput = (providers) => ({ ...providers, capabilities: expandProviderCapabilities( @@ -97,7 +182,11 @@ export const expandProviderOutput = (providers) => ({ ), }); -/** Expand the CLI capabilities command's compact provider descriptor groups. */ +/** + * Expand the CLI capabilities command's compact provider descriptor groups. + * @template {{ readonly capabilities: unknown }} Capabilities + * @param {Capabilities} capabilities + */ export const expandCapabilitiesOutput = (capabilities) => ({ ...capabilities, capabilities: expandProviderCapabilities( @@ -105,76 +194,157 @@ export const expandCapabilitiesOutput = (capabilities) => ({ ), }); -/** Validate an Evidence-producing tool output and return its normalized result. */ +/** + * Validate an Evidence-producing tool output and return its normalized result. + * @template {ToolName} N + * @param {CallToolResult} result + * @param {N} operation + * @returns {OperationResult>} + */ export const requireMcpEvidenceResult = (result, operation) => { const contract = toolContract(operation); if (operationResultKey(contract) !== "normalized_result") throw new Error(`${operation} requires the lifecycle result extractor`); - return requireMcpOutput(result, operation, contract).normalized_result; + return projectOperationResult( + requireMcpOutput(result, operation, contract), + "normalized_result", + ); }; -/** Validate a lifecycle tool output and return its operation result. */ +/** + * Validate a lifecycle tool output and return its operation result. + * @template {ToolName} N + * @param {CallToolResult} result + * @param {N} operation + * @returns {VerifierOperationResult} + */ export const requireMcpLifecycleResult = (result, operation) => { const contract = toolContract(operation); if (operationResultKey(contract) !== "result") throw new Error(`${operation} requires the Evidence result extractor`); - const projected = requireMcpOutput(result, operation, contract).result; - return operation === "binary_session" - ? expandMcpSessionStatus(projected) - : projected; + return expandOperationResult( + operation, + projectOperationResult( + requireMcpOutput(result, operation, contract), + "result", + ), + ); }; -/** Require an authenticated complete bundle without changing summary-capable readers. */ +/** + * Require an authenticated complete bundle without changing summary-capable readers. + * @param {CallToolResult} result + */ export const requireMcpEvidenceBundle = (result) => { - const operation = "get_evidence_bundle"; - const contract = toolContract(operation); - const bundle = requireMcpOutput(result, operation, contract).result; - if (bundle.kind === "evidence-bundle-summary") + const bundle = requireMcpOutput(result, "get_evidence_bundle").result; + if ("kind" in bundle) throw new Error( "get_evidence_bundle returned summary metadata instead of complete Evidence", ); return authenticateParsedEvidenceBundle(bundle); }; -/** Project a mixed verifier call using its exact named canonical output schema. */ +/** + * Project a mixed verifier call using its exact named canonical output schema. + * @template {ToolName} N + * @param {CallToolResult} result + * @param {N} operation + * @param {NamedToolContract} [contract] + * @returns {VerifierOperationResult} + */ export const requireMcpOperationResult = ( result, operation, contract = toolContract(operation), -) => { - const key = operationResultKey(contract); - const projected = requireMcpOutput(result, operation, contract)[key]; - return operation === "binary_session" - ? expandMcpSessionStatus(projected) - : projected; -}; +) => + expandOperationResult( + operation, + projectOperationResult( + requireMcpOutput(result, operation, contract), + operationResultKey(contract), + ), + ); -/** Verify provider identity directly from the tool result. */ +/** + * Read the member that operationResultKey selected from the contract. + * @template {PublicToolContract} C + * @param {ToolOutput} output + * @param {"normalized_result" | "result"} key + * @returns {OperationResult} + */ +const projectOperationResult = (output, key) => + /** @type {Record>} */ (output)[key]; + +/** + * @template {ToolName} N + * @param {N} operation + * @param {OperationResult>} projected + * @returns {VerifierOperationResult} + */ +const expandOperationResult = (operation, projected) => + /** @type {VerifierOperationResult} */ ( + operation === "binary_session" + ? expandMcpSessionStatus(/** @type {SessionStatus} */ (projected)) + : projected + ); + +const providerReference = z.object({ id: z.string() }); +const versionedProviderReference = z.object({ + id: z.string(), + version: z.string(), +}); +const providerProvenance = z.object({ + provider: providerReference, + analysis_profile: z.object({ provider: versionedProviderReference }), +}); +const workflowProvenance = z.object({ + provider: providerReference, + analysis_profile: z.object({ + provider: providerReference, + parameters: z.object({ + upstream_analysis_profile: z.object({ + provider: versionedProviderReference, + }), + }), + }), +}); + +/** + * Verify provider identity directly from the tool result. + * @param {CallToolResult} result + * @param {string} operation + * @param {string} providerId + */ export const requireEvidenceProvider = (result, operation, providerId) => { - const evidence = jsonValue(result); + const evidence = providerProvenance.safeParse(jsonValue(result)); if ( - evidence?.provider?.id !== providerId || - evidence?.analysis_profile?.provider?.id !== providerId || - typeof evidence.analysis_profile.provider.version !== "string" + !evidence.success || + evidence.data.provider.id !== providerId || + evidence.data.analysis_profile.provider.id !== providerId ) { throw new Error(`${operation} omitted its concrete provider provenance`); } }; -/** Verify composed workflow and upstream provider provenance inline. */ +/** + * Verify composed workflow and upstream provider provenance inline. + * @param {CallToolResult} result + * @param {string} operation + * @param {{ readonly workflowProviderId: string; readonly upstreamProviderId: string }} expected + */ export const requireWorkflowEvidenceProvider = ( result, operation, expected, ) => { - const evidence = jsonValue(result); - const profile = evidence?.analysis_profile; - const upstream = profile?.parameters?.upstream_analysis_profile; + const evidence = workflowProvenance.safeParse(jsonValue(result)); if ( - evidence?.provider?.id !== expected.workflowProviderId || - profile?.provider?.id !== expected.workflowProviderId || - upstream?.provider?.id !== expected.upstreamProviderId || - typeof upstream.provider.version !== "string" + !evidence.success || + evidence.data.provider.id !== expected.workflowProviderId || + evidence.data.analysis_profile.provider.id !== + expected.workflowProviderId || + evidence.data.analysis_profile.parameters.upstream_analysis_profile.provider + .id !== expected.upstreamProviderId ) { throw new Error( `${operation} omitted its composed workflow or upstream provenance`, diff --git a/scripts/lib/real-ghidra-snapshot-lifecycle.mjs b/scripts/lib/real-ghidra-snapshot-lifecycle.mjs index 219c69c000..8c5b64f3c2 100644 --- a/scripts/lib/real-ghidra-snapshot-lifecycle.mjs +++ b/scripts/lib/real-ghidra-snapshot-lifecycle.mjs @@ -6,10 +6,58 @@ import assert from "node:assert/strict"; import { access, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { parseAnalysisSnapshot } from "../../dist/domain/analysisSnapshot.js"; -import { parseEvidenceBundle } from "../../dist/domain/evidenceBundle.js"; +import { parseAnalysisSnapshot } from "#dist/domain/analysisSnapshot.js"; +import { parseEvidenceBundle } from "#dist/domain/evidenceBundle.js"; +import { toolContract } from "#dist/contracts/toolContracts.js"; -/** Keep live Ghidra edits distinct from historical Evidence across snapshots. */ +/** @typedef {import("@modelcontextprotocol/client").Client} Client */ +/** @typedef {import("./mcp-verifier-results.mjs").ToolName} ToolName */ +/** + * @template {ToolName} N + * @typedef {import("./mcp-verifier-results.mjs").VerifierOperationResult} VerifierOperationResult + */ +/** + * The complete analyze_function dossier; a retained summary view has no + * procedure. + * @typedef {Extract, { readonly procedure: unknown }>} FunctionDossier + */ +/** + * Call one tool and return its contract-validated operation result. + * @typedef {(name: N, args?: Record) => Promise>} CallTool + */ +/** + * Require a snapshot binding rejection from one tool call. + * @typedef {(name: ToolName, args: Record) => Promise} RejectSnapshot + */ + +/** + * Require the complete dossier that `detail: "complete"` selects. + * @param {VerifierOperationResult<"analyze_function">} result + * @returns {FunctionDossier} + */ +const completeDossier = (result) => { + assert.ok( + "procedure" in result, + "analyze_function returned a summary view instead of its dossier", + ); + return result; +}; + +// Bundle records keep each operation's normalized result as JSON; read it +// through that operation's own contract. +const annotationResult = toolContract("annotate_native_function").outputSchema + .shape.normalized_result; +const nativeApiResult = + toolContract("inspect_native_api").outputSchema.shape.normalized_result; + +/** + * Keep live Ghidra edits distinct from historical Evidence across snapshots. + * @param {Client} client + * @param {{ readonly path: string; readonly sha256: string }} target + * @param {FunctionDossier} edited The MCP session's dossier after its edit. + * @param {(command: "function", value: string, flags?: readonly string[]) => Promise} cli + * Runs a one-shot CLI command against its own unmodified database. + */ export async function verifyGhidraSnapshotLifecycle( client, target, @@ -19,6 +67,7 @@ export async function verifyGhidraSnapshotLifecycle( const workspace = await mkdtemp(join(tmpdir(), "rea-ghidra-snapshot-proof-")); let successfulCalls = 0; let rejectedCalls = 0; + /** @type {CallTool} */ const call = async (name, args = {}) => { const result = requireMcpOperationResult( await client.callTool({ name, arguments: args }, { timeout: 180000 }), @@ -27,6 +76,7 @@ export async function verifyGhidraSnapshotLifecycle( successfulCalls++; return result; }; + /** @type {RejectSnapshot} */ const rejectSnapshot = async (name, args) => { const reply = await client.callTool({ name, arguments: args }); assert.equal( @@ -43,6 +93,11 @@ export async function verifyGhidraSnapshotLifecycle( assert.equal(error.retryable, false); rejectedCalls++; }; + const currentRunId = async () => { + const { analysis_run: run } = await call("binary_session"); + assert.ok(run !== null, "binary_session omitted the open analysis run"); + return run.run_id; + }; try { const entry = edited.procedure.address; const snapshotPath = join(workspace, "original.json"); @@ -54,7 +109,7 @@ export async function verifyGhidraSnapshotLifecycle( ); assert.equal(snapshot.target.identity.sha256, target.sha256); assert.notEqual(original.procedure.name, edited.procedure.name); - const runId = (await call("binary_session")).analysis_run.run_id; + const runId = await currentRunId(); const api = await call("inspect_native_api", { procedure: entry }); assert.equal(api.procedure.name, edited.procedure.name); for (const repeatedOpen of [false, true]) { @@ -66,7 +121,7 @@ export async function verifyGhidraSnapshotLifecycle( ); await rejectSnapshot("close_binary", { snapshot_path: destination }); await assert.rejects(access(destination), { code: "ENOENT" }); - assert.equal((await call("binary_session")).analysis_run.run_id, runId); + assert.equal(await currentRunId(), runId); assert.deepEqual( await call("analyze_function", { procedure: entry, @@ -81,7 +136,7 @@ export async function verifyGhidraSnapshotLifecycle( snapshot_path: snapshotPath, }); assert.deepEqual(await readFile(snapshotPath), snapshotBytes); - assert.equal((await call("binary_session")).analysis_run.run_id, runId); + assert.equal(await currentRunId(), runId); assert.deepEqual( await call("analyze_function", { procedure: entry, detail: "complete" }), edited, @@ -93,19 +148,22 @@ export async function verifyGhidraSnapshotLifecycle( const bundle = parseEvidenceBundle( JSON.parse(await readFile(bundlePath, "utf8")), ); - const annotation = bundle.records.find( - (record) => - record.operation === "annotate_native_function" && - record.normalized_result.annotations.name === edited.procedure.name && - record.normalized_result.annotations.address === - edited.procedure.address, - ); - const editedApi = bundle.records.find( - (record) => - record.operation === "inspect_native_api" && - record.normalized_result.procedure.name === edited.procedure.name && - record.normalized_result.procedure.address === edited.procedure.address, - ); + const annotation = bundle.records.find((record) => { + if (record.operation !== "annotate_native_function") return false; + const { annotations } = annotationResult.parse(record.normalized_result); + return ( + annotations.name === edited.procedure.name && + annotations.address === edited.procedure.address + ); + }); + const editedApi = bundle.records.find((record) => { + if (record.operation !== "inspect_native_api") return false; + const { procedure } = nativeApiResult.parse(record.normalized_result); + return ( + procedure.name === edited.procedure.name && + procedure.address === edited.procedure.address + ); + }); assert.ok(annotation); assert.ok(editedApi); @@ -115,13 +173,12 @@ export async function verifyGhidraSnapshotLifecycle( provider_id: "ghidra", snapshot_path: snapshotPath, }); - const restored = await call("analyze_function", { - procedure: entry, - detail: "complete", - }); + const restored = completeDossier( + await call("analyze_function", { procedure: entry, detail: "complete" }), + ); assert.deepEqual(restored.procedure, original.procedure); assert.deepEqual(restored.comments, original.comments); - assert.notEqual((await call("binary_session")).analysis_run.run_id, runId); + assert.notEqual(await currentRunId(), runId); const freshPath = join(workspace, "fresh.json"); await call("close_binary", { snapshot_path: freshPath }); const fresh = parseAnalysisSnapshot( @@ -153,6 +210,16 @@ export async function verifyGhidraSnapshotLifecycle( } } +/** + * @param {{ + * readonly call: CallTool; + * readonly rejectSnapshot: RejectSnapshot; + * readonly workspace: string; + * readonly target: { readonly path: string }; + * readonly entry: string; + * readonly original: FunctionDossier; + * }} context + */ async function verifyConcurrentSnapshotClose({ call, rejectSnapshot, @@ -180,8 +247,9 @@ async function verifyConcurrentSnapshotClose({ await call("close_binary"); await call("open_binary", { path: target.path, provider_id: "ghidra" }); assert.deepEqual( - (await call("analyze_function", { procedure: entry, detail: "complete" })) - .procedure, + completeDossier( + await call("analyze_function", { procedure: entry, detail: "complete" }), + ).procedure, original.procedure, ); } diff --git a/scripts/lib/verifier-artifact-scope.mjs b/scripts/lib/verifier-artifact-scope.mjs index 7075e614c5..0100fe4e16 100644 --- a/scripts/lib/verifier-artifact-scope.mjs +++ b/scripts/lib/verifier-artifact-scope.mjs @@ -1,7 +1,12 @@ -import { ArtifactResourceScope } from "../../dist/resources/ArtifactResourceScope.js"; +import { ArtifactResourceScope } from "#dist/resources/ArtifactResourceScope.js"; /** A failed close retains the scope that can retry all of its owners. */ export class VerifierArtifactCleanupFailure extends AggregateError { + /** + * @param {ArtifactResourceScope} scope + * @param {readonly unknown[]} failures + * @param {unknown} cause + */ constructor(scope, failures, cause) { super(failures, "Verifier cleanup failed", { cause }); this._tag = "VerifierArtifactCleanupFailure"; @@ -9,9 +14,15 @@ export class VerifierArtifactCleanupFailure extends AggregateError { } } -/** Run one verifier operation and retain its exact scope if close fails. */ +/** + * Run one verifier operation and retain its exact scope if close fails. + * @template T + * @param {(scope: ArtifactResourceScope) => Promise} operation + * @returns {Promise} + */ export async function withVerifierArtifactScope(operation) { const scope = new ArtifactResourceScope(); + /** @type {{ readonly ok: true; readonly value: T } | { readonly ok: false; readonly cause: unknown }} */ let outcome; try { outcome = { ok: true, value: await operation(scope) }; diff --git a/scripts/verify-package-evidence.mjs b/scripts/verify-package-evidence.mjs index 194c3e8932..b5899e8c00 100644 --- a/scripts/verify-package-evidence.mjs +++ b/scripts/verify-package-evidence.mjs @@ -1,10 +1,29 @@ import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { createEvidence } from "../dist/domain/evidence.js"; -import { unknownTarget } from "../dist/domain/targetIdentity.js"; +import { createEvidence } from "#dist/domain/evidence.js"; +import { + createEvidenceBundle, + serializeEvidenceBundle, +} from "#dist/domain/evidenceBundle.js"; +import { unknownTarget } from "#dist/domain/targetIdentity.js"; -/** Round-trip historical reference import and evidence bundles. */ +/** + * Round-trip historical reference import and evidence bundles. + * @param {{ + * readonly commands: { + * run( + * command: string, + * arguments_: readonly string[], + * environment?: NodeJS.ProcessEnv, + * ): Promise; + * }; + * readonly cli: string; + * readonly evidenceRoot: string; + * readonly referenceRoot: string; + * readonly environment: NodeJS.ProcessEnv; + * }} options + */ export async function verifyPackageEvidence({ commands, cli, @@ -26,9 +45,6 @@ export async function verifyPackageEvidence({ JSON.stringify(referenceImport).includes("PACKAGE_SECRET_SENTINEL") ) throw new Error("packaged historical reference import CLI failed"); - const { createEvidenceBundle, serializeEvidenceBundle } = await import( - new URL("../dist/domain/evidenceBundle.js", import.meta.url) - ); const sourceBundle = createEvidenceBundle([ createEvidence( unknownTarget("not-observed"), diff --git a/scripts/verify/javascript/digests.mjs b/scripts/verify/javascript/digests.mjs index 912fb39e8f..3987b7f36d 100644 --- a/scripts/verify/javascript/digests.mjs +++ b/scripts/verify/javascript/digests.mjs @@ -4,12 +4,9 @@ import assert from "node:assert/strict"; import { constants } from "node:buffer"; import { createHash } from "node:crypto"; -import { digestCanonicalValue } from "../../../dist/domain/canonicalDigest.js"; -import { - createEvidence, - parseEvidence, -} from "../../../dist/domain/evidence.js"; -import { unknownTarget } from "../../../dist/domain/targetIdentity.js"; +import { digestCanonicalValue } from "#dist/domain/canonicalDigest.js"; +import { createEvidence, parseEvidence } from "#dist/domain/evidence.js"; +import { unknownTarget } from "#dist/domain/targetIdentity.js"; // This opt-in check crosses the actual engine string limit; keep it outside // routine source tests. Build first, then run this script with the local Node. diff --git a/scripts/verify/javascript/output.mjs b/scripts/verify/javascript/output.mjs index 5c1eed600b..83bce7ce70 100644 --- a/scripts/verify/javascript/output.mjs +++ b/scripts/verify/javascript/output.mjs @@ -13,10 +13,10 @@ import { finished } from "node:stream/promises"; import { Cli } from "incur"; import { estimateTokenCount } from "tokenx"; -import { parseCliOutputArguments } from "../../../dist/cliOutput.js"; -import { createCliResultOutput } from "../../../dist/cli/cliResultOutput.js"; -import { createEvidence } from "../../../dist/domain/evidence.js"; -import { unknownTarget } from "../../../dist/domain/targetIdentity.js"; +import { parseCliOutputArguments } from "#dist/cliOutput.js"; +import { createCliResultOutput } from "#dist/cli/cliResultOutput.js"; +import { createEvidence } from "#dist/domain/evidence.js"; +import { unknownTarget } from "#dist/domain/targetIdentity.js"; import { VerifierFixture, withVerifierFixture, @@ -39,6 +39,7 @@ const fixture = await VerifierFixture.open({ }); const workspace = fixture.path; const path = join(workspace, "result.json"); +/** @type {import("node:fs").WriteStream | undefined} */ let destination; let peakRss = process.memoryUsage().rss; const sampleRss = () => { diff --git a/scripts/verify/managed/completion-report.mjs b/scripts/verify/managed/completion-report.mjs index 1f038b547e..9aeab0b786 100644 --- a/scripts/verify/managed/completion-report.mjs +++ b/scripts/verify/managed/completion-report.mjs @@ -1,5 +1,33 @@ -import { createEvidence } from "../../../dist/domain/evidence.js"; -import { unknownTarget } from "../../../dist/domain/targetIdentity.js"; +import { createEvidence } from "#dist/domain/evidence.js"; +import { unknownTarget } from "#dist/domain/targetIdentity.js"; + +/** @typedef {import("#dist/domain/binaryTargetTypes.js").BinaryTarget} BinaryTarget */ +/** @typedef {import("#dist/domain/evidence.js").EvidenceSubjectTarget} EvidenceSubjectTarget */ +/** + * A verified file as the manifest verifier and ILSpy oracle report it. + * @typedef {{ readonly file_name: string; readonly sha256: string }} ReportedTarget + */ +/** + * A source-owned fixture the conformance run wrote and resolved. + * @typedef {{ readonly target: BinaryTarget }} ResolvedFixture + */ +/** + * The completed conformance observations the claims are projected from. + * @typedef {{ + * readonly modern: ResolvedFixture; + * readonly framework: ResolvedFixture; + * readonly nativeFunctionTarget: EvidenceSubjectTarget; + * readonly readyToRun: ResolvedFixture; + * readonly obfuscated: ResolvedFixture; + * readonly left: ResolvedFixture; + * readonly right: ResolvedFixture; + * readonly nativeOnly: ResolvedFixture; + * readonly malformed: ResolvedFixture; + * readonly manifestSelfTest: { readonly target: ReportedTarget }; + * readonly operatorManifest: { readonly target: ReportedTarget } | null; + * readonly ilspyOracle: { readonly fixture: ReportedTarget } | null; + * }} ManagedCompletionReportInput + */ const COMPLETION_PROVIDER = { id: "rea-managed-conformance", @@ -7,6 +35,15 @@ const COMPLETION_PROVIDER = { version: "1", }; +/** + * @param {{ + * readonly claimId: string; + * readonly scenarioId: string; + * readonly targets?: readonly (BinaryTarget | EvidenceSubjectTarget)[]; + * readonly status?: "pass" | "unsupported"; + * readonly limitations?: readonly string[]; + * }} claim + */ const completionClaim = ({ claimId, scenarioId, @@ -48,11 +85,16 @@ const completionClaim = ({ }; }; +/** + * @param {ReportedTarget | undefined} target + * @returns {EvidenceSubjectTarget[]} + */ const optionalTarget = (target) => target === undefined ? [] : [{ path: target.file_name, sha256: target.sha256, format: "pe" }]; +/** @param {ManagedCompletionReportInput} input */ const sourceOwnedClaims = ({ modern, framework, @@ -128,6 +170,7 @@ const sourceOwnedClaims = ({ }), ]; +/** @param {ManagedCompletionReportInput} input */ const optionalClaims = ({ operatorManifest, ilspyOracle }) => [ completionClaim({ claimId: "managed.operator-local-manifest", @@ -149,7 +192,11 @@ const optionalClaims = ({ operatorManifest, ilspyOracle }) => [ }), ]; -/** Project completed managed assertions into the standard verifier report. */ +/** + * Project completed managed assertions into the standard verifier report. + * @param {ManagedCompletionReportInput} input + * @param {unknown} verifierRun The completed run record, reported unchanged. + */ export const createManagedCompletionReport = (input, verifierRun) => ({ verifier: { id: COMPLETION_PROVIDER.id, diff --git a/scripts/verify/managed/manifest-parser.mjs b/scripts/verify/managed/manifest-parser.mjs index 6c11aedcd9..2628cffdcc 100644 --- a/scripts/verify/managed/manifest-parser.mjs +++ b/scripts/verify/managed/manifest-parser.mjs @@ -1,8 +1,18 @@ +import { isRecord } from "#dist/domain/isRecord.js"; + +/** @typedef {ReturnType} ManagedAppManifest */ +/** @typedef {ManagedAppManifest["methods"][number]} ManagedAppManifestMethod */ +/** @typedef {NonNullable} ManagedAppManifestGraph */ + +/** + * Parse a caller-selected managed app manifest into its checked expectations. + * @param {unknown} rawManifest + */ export function parseManagedAppManifest(rawManifest) { const manifest = object(rawManifest, "manifest"); const target = object(manifest.target, "manifest.target"); const methods = array(manifest.methods, "manifest.methods"); - ensure(manifest.methods.length > 0, "manifest.methods must not be empty"); + ensure(methods.length > 0, "manifest.methods must not be empty"); return { label: optionalString(manifest.label, "manifest.label"), target: { @@ -29,6 +39,7 @@ export function parseManagedAppManifest(rawManifest) { }; } +/** @param {unknown} rawGraph */ function parseManagedAppManifestGraph(rawGraph) { if (rawGraph === undefined) return undefined; const graph = object(rawGraph, "manifest.application_graph"); @@ -61,6 +72,10 @@ function parseManagedAppManifestGraph(rawGraph) { }; } +/** + * @param {unknown} rawTrace + * @param {number} index + */ function parseManagedAppManifestTrace(rawTrace, index) { const prefix = `manifest.application_graph.feature_traces[${String(index)}]`; const trace = object(rawTrace, prefix); @@ -76,7 +91,9 @@ function parseManagedAppManifestTrace(rawTrace, index) { ); const minMatchedSeeds = trace.min_matched_seeds ?? 1; ensure( - Number.isInteger(minMatchedSeeds) && minMatchedSeeds > 0, + typeof minMatchedSeeds === "number" && + Number.isInteger(minMatchedSeeds) && + minMatchedSeeds > 0, `${prefix}.min_matched_seeds must be a positive integer`, ); return { @@ -89,6 +106,10 @@ function parseManagedAppManifestTrace(rawTrace, index) { }; } +/** + * @param {unknown} rawMethod + * @param {number} index + */ function parseManagedAppManifestMethod(rawMethod, index) { const prefix = `manifest.methods[${String(index)}]`; const method = object(rawMethod, prefix); @@ -112,19 +133,31 @@ function parseManagedAppManifestMethod(rawMethod, index) { }; } +/** + * @param {unknown} value + * @param {string} name + * @returns {{ readonly [key: string]: unknown }} + */ function object(value, name) { - ensure( - value !== null && typeof value === "object" && !Array.isArray(value), - `${name} must be an object`, - ); + ensure(isRecord(value), `${name} must be an object`); return value; } +/** + * @param {unknown} value + * @param {string} name + * @returns {readonly unknown[]} + */ function array(value, name) { ensure(Array.isArray(value), `${name} must be an array`); return value; } +/** + * @param {unknown} value + * @param {string} name + * @returns {string} + */ function string(value, name) { ensure( typeof value === "string" && value.length > 0, @@ -133,25 +166,42 @@ function string(value, name) { return value; } +/** + * @param {unknown} value + * @param {string} name + */ function optionalString(value, name) { if (value === undefined) return undefined; return string(value, name); } +/** + * @param {unknown} value + * @param {string} name + * @returns {number} + */ function nonNegativeInteger(value, name) { ensure( - Number.isInteger(value) && value >= 0, + typeof value === "number" && Number.isInteger(value) && value >= 0, `${name} must be a non-negative integer`, ); return value; } +/** + * @param {unknown} value + * @param {string} name + */ function digest(value, name) { const text = string(value, name); ensure(/^[a-f0-9]{64}$/u.test(text), `${name} must be a lowercase sha256`); return text; } +/** + * @param {unknown} value + * @param {string} name + */ function optionalCliMetadataGuid(value, name) { if (value === undefined) return undefined; if (value === null) return null; @@ -165,12 +215,20 @@ function optionalCliMetadataGuid(value, name) { return text; } +/** + * @param {unknown} value + * @param {string} name + */ function metadataToken(value, name) { const text = string(value, name); ensure(/^0x[0-9a-f]{8}$/u.test(text), `${name} must be a metadata token`); return text; } +/** + * @param {unknown} value + * @param {string} name + */ function methodToken(value, name) { const token = metadataToken(value, name); ensure( @@ -180,12 +238,21 @@ function methodToken(value, name) { return token; } +/** + * The MethodDef row a checked method token names. + * @param {string} token + */ export function methodTokenRow(token) { const row = Number.parseInt(token.slice(4), 16); ensure(row > 0, `method token ${token} has an invalid row id`); return row; } +/** + * @param {unknown} condition + * @param {string} message + * @returns {asserts condition} + */ export function ensure(condition, message) { if (!condition) throw new Error(`managed app manifest verification failed: ${message}`); diff --git a/scripts/verify/managed/manifest.mjs b/scripts/verify/managed/manifest.mjs index 77d82f0571..3efe576581 100644 --- a/scripts/verify/managed/manifest.mjs +++ b/scripts/verify/managed/manifest.mjs @@ -2,26 +2,60 @@ import { basename, dirname, isAbsolute, resolve } from "node:path"; import { readManagedSnapshot, verifyManagedSourceIdentity, -} from "../../../dist/dotnet/ManagedSnapshot.js"; +} from "#dist/dotnet/ManagedSnapshot.js"; import { withVerifierArtifactScope } from "../../lib/verifier-artifact-scope.mjs"; -import { parseBinaryTarget } from "../../../dist/application/BinaryTargetResolver.js"; -import { inspectManagedArtifactBytes } from "../../../dist/dotnet/ManagedArtifactInspector.js"; -import { inspectManagedMembersBytes } from "../../../dist/dotnet/ManagedMemberInspector.js"; -import { projectManagedApplicationGraphEvidence } from "../../../dist/application/managed/ManagedApplicationGraphService.js"; -import { traceApplicationFeatureEvidence } from "../../../dist/application/javascript/JavaScriptApplicationWorkflowService.js"; -import { createEvidence } from "../../../dist/domain/evidence.js"; -import { MANAGED_STATIC_PROVIDER } from "../../../dist/domain/investigationProviders.js"; +import { parseBinaryTarget } from "#dist/application/BinaryTargetResolver.js"; +import { inspectManagedArtifactBytes } from "#dist/dotnet/ManagedArtifactInspector.js"; +import { inspectManagedMembersBytes } from "#dist/dotnet/ManagedMemberInspector.js"; +import { projectManagedApplicationGraphEvidence } from "#dist/application/managed/ManagedApplicationGraphService.js"; +import { traceApplicationFeatureEvidence } from "#dist/application/javascript/JavaScriptApplicationWorkflowService.js"; +import { createEvidence } from "#dist/domain/evidence.js"; +import { MANAGED_STATIC_PROVIDER } from "#dist/domain/investigationProviders.js"; +import { managedApplicationGraphResultSchema } from "#dist/domain/managed/managedApplicationGraph.js"; import { ensure, methodTokenRow, parseManagedAppManifest, } from "./manifest-parser.mjs"; +/** @typedef {import("#dist/domain/binaryTargetTypes.js").BinaryTarget} BinaryTarget */ +/** @typedef {import("#dist/resources/ArtifactResourceScope.js").ArtifactResourceScope} ArtifactResourceScope */ +/** @typedef {import("#dist/domain/evidence.js").Evidence} Evidence */ +/** @typedef {ReturnType} ManagedArtifactInspection */ +/** @typedef {ReturnType["methods"][number]} ManagedMethod */ +/** @typedef {import("./manifest-parser.mjs").ManagedAppManifest} ManagedAppManifest */ +/** @typedef {import("./manifest-parser.mjs").ManagedAppManifestMethod} ManagedAppManifestMethod */ +/** @typedef {import("./manifest-parser.mjs").ManagedAppManifestGraph} ManagedAppManifestGraph */ +/** + * One manifest method's application graph projection. + * @typedef {{ + * readonly method: ManagedMethod; + * readonly evidence: Evidence; + * readonly nodeKinds: readonly string[]; + * readonly summary: import("zod").output["summary"]; + * }} ManifestMethodProjection + */ +/** + * Inputs shared by the graph projections of one verified target. + * @typedef {{ + * readonly graphByToken: Map; + * readonly inspectedMethods: ReadonlyMap; + * readonly bytes: Buffer; + * readonly target: BinaryTarget; + * readonly artifact: ManagedArtifactInspection; + * }} ProjectionState + */ + // Preserve fs.readFile admission: Node internal kIoMaxLength is 2 GiB minus one. // https://nodejs.org/api/errors.html#err_fs_file_too_large const READ_FILE_MAX_BYTES = 0x7fff_ffff; +/** + * Verify a caller-selected managed app manifest against its target. + * @param {unknown} rawManifest + * @param {string} manifestPath Resolves a relative target path. + */ export async function verifyManagedAppManifest(rawManifest, manifestPath) { return withVerifierArtifactScope((resources) => resources.run(() => @@ -30,6 +64,11 @@ export async function verifyManagedAppManifest(rawManifest, manifestPath) { ); } +/** + * @param {unknown} rawManifest + * @param {string} manifestPath + * @param {ArtifactResourceScope} resources + */ async function verifyManifestSnapshot(rawManifest, manifestPath, resources) { const manifest = parseManagedAppManifest(rawManifest); const targetPath = isAbsolute(manifest.target.path) @@ -93,6 +132,10 @@ async function verifyManifestSnapshot(rawManifest, manifestPath, resources) { }; } +/** + * @param {ManagedAppManifest} manifest + * @param {ManagedArtifactInspection} artifact + */ function verifyManifestTarget(manifest, artifact) { ensure( artifact.classification.status === "managed", @@ -121,9 +164,15 @@ function verifyManifestTarget(manifest, artifact) { ); } +/** + * @param {ManagedAppManifest} manifest + * @param {Buffer} bytes + * @param {BinaryTarget} target + */ function verifyManifestMethods(manifest, bytes, target) { const methods = []; let assertions = 0; + /** @type {Map} */ const inspectedMethods = new Map(); for (const expectedMethod of manifest.methods) { const method = inspectManagedMethodByToken(bytes, target, expectedMethod); @@ -160,6 +209,16 @@ function verifyManifestMethods(manifest, bytes, target) { return { methods, inspectedMethods, assertions }; } +/** + * @param {{ + * readonly graphManifest: ManagedAppManifestGraph; + * readonly bytes: Buffer; + * readonly target: BinaryTarget; + * readonly artifact: ManagedArtifactInspection; + * readonly manifestMethods: readonly ManagedAppManifestMethod[]; + * readonly inspectedMethods: ReadonlyMap; + * }} options + */ function verifyManagedAppManifestApplicationGraph({ graphManifest, bytes, @@ -168,6 +227,7 @@ function verifyManagedAppManifestApplicationGraph({ manifestMethods, inspectedMethods, }) { + /** @type {Map} */ const graphByToken = new Map(); const projectionState = { graphByToken, @@ -242,6 +302,11 @@ function verifyManagedAppManifestApplicationGraph({ }; } +/** + * @param {ProjectionState} state + * @param {string} token + * @returns {ManifestMethodProjection} + */ const projectManifestMethod = (state, token) => { const existing = state.graphByToken.get(token); if (existing !== undefined) return existing; @@ -286,20 +351,25 @@ const projectManifestMethod = (state, token) => { projected.ok, `application graph projection failed for method ${token}`, ); + // Read the projection through the managed graph owner's result schema. + const graph = managedApplicationGraphResultSchema.parse( + projected.value.normalized_result, + ); const projection = { method, evidence: projected.value, - nodeKinds: [ - ...new Set( - projected.value.normalized_result.graph.nodes.map(({ kind }) => kind), - ), - ].sort(), - summary: projected.value.normalized_result.summary, + nodeKinds: [...new Set(graph.graph.nodes.map(({ kind }) => kind))].sort(), + summary: graph.summary, }; state.graphByToken.set(token, projection); return projection; }; +/** + * @param {Buffer} bytes + * @param {BinaryTarget} target + * @param {ManagedAppManifestMethod} expectedMethod + */ function inspectManagedMethodByToken(bytes, target, expectedMethod) { const members = inspectManagedMethodPageByToken( bytes, @@ -320,6 +390,11 @@ function inspectManagedMethodByToken(bytes, target, expectedMethod) { return method; } +/** + * @param {Buffer} bytes + * @param {BinaryTarget} target + * @param {string} token + */ function inspectManagedMethodPageByToken(bytes, target, token) { const row = methodTokenRow(token); return inspectManagedMembersBytes(bytes, target, { diff --git a/scripts/verify/managed/oracles.mjs b/scripts/verify/managed/oracles.mjs index 8d90e0e472..eca2b80454 100644 --- a/scripts/verify/managed/oracles.mjs +++ b/scripts/verify/managed/oracles.mjs @@ -5,15 +5,46 @@ import { basename, isAbsolute, join, resolve } from "node:path"; import assert from "node:assert/strict"; import { promisify } from "node:util"; -import { inspectManagedArtifactBytes } from "../../../dist/dotnet/ManagedArtifactInspector.js"; -import { inspectManagedMembersBytes } from "../../../dist/dotnet/ManagedMemberInspector.js"; -import { importManagedReconstructionEvidence } from "../../../dist/application/managed/ManagedReconstructionService.js"; -import { createEvidence } from "../../../dist/domain/evidence.js"; -import { MANAGED_STATIC_PROVIDER } from "../../../dist/domain/investigationProviders.js"; +import { inspectManagedArtifactBytes } from "#dist/dotnet/ManagedArtifactInspector.js"; +import { inspectManagedMembersBytes } from "#dist/dotnet/ManagedMemberInspector.js"; +import { importManagedReconstructionEvidence } from "#dist/application/managed/ManagedReconstructionService.js"; +import { createEvidence } from "#dist/domain/evidence.js"; +import { MANAGED_STATIC_PROVIDER } from "#dist/domain/investigationProviders.js"; +import { managedReconstructionImportResultSchema } from "#dist/domain/managed/managedReconstruction.js"; import { verifyManagedAppManifest } from "./manifest.mjs"; +/** @typedef {import("#dist/domain/binaryTargetTypes.js").BinaryTarget} BinaryTarget */ +/** @typedef {import("#dist/domain/evidence.js").Evidence} Evidence */ +/** @typedef {ReturnType} ManagedMemberInspection */ +/** @typedef {ManagedMemberInspection["methods"][number]} ManagedMethod */ + +/** + * A source-owned managed fixture written into the verifier workspace. + * @typedef {{ + * readonly bytes: Buffer; + * readonly path: string; + * readonly target: BinaryTarget; + * }} ManagedOracleFixture + */ + +/** + * What the oracles need from the conformance support. + * @typedef {{ + * readonly workspace: string; + * readonly defaultIlBody: Buffer; + * readonly fixture: ( + * name: string, + * options: { readonly methodName: string; readonly ilBody?: Buffer }, + * ) => Promise; + * readonly sha256: (bytes: Buffer) => string; + * }} ManagedOracleContext + */ + +/** @typedef {ManagedOracleContext & { readonly verifyManagedAppManifest: typeof verifyManagedAppManifest }} ManagedOracleSupport */ + const execFileAsync = promisify(execFile); +/** @param {ManagedOracleContext} context */ export const createManagedConformanceOracleSupport = (context) => { const support = { ...context, verifyManagedAppManifest }; return { @@ -23,6 +54,7 @@ export const createManagedConformanceOracleSupport = (context) => { }; }; +/** @param {ManagedOracleSupport} context */ async function runManagedAppManifestSelfTest(context) { const sample = await context.fixture("manifest-self-test.exe", { methodName: "PinnedSemanticSlice", @@ -94,45 +126,60 @@ async function runManagedAppManifestSelfTest(context) { }; } +/** + * Each authored mismatch must be refused by the manifest verifier. + * @param {ManagedOracleSupport} context + * @param {object} manifest A manifest the verifier accepts. + * @param {string} manifestPath + */ async function verifyManifestRejections(context, manifest, manifestPath) { const refusals = [ - [ - "target sha256", - ["target", "sha256"], - "0".repeat(64), - /target sha256 mismatch/u, - ], - [ - "method token", - ["methods", 0, "token"], - "0x0600ffff", - /method 0x0600ffff was not returned/u, - ], - [ - "signature sha256", - ["methods", 0, "signature_sha256"], - "0".repeat(64), - /signature sha256 mismatch/u, - ], - ["IL size", ["methods", 0, "il_size"], 999, /IL size mismatch/u], - [ - "IL sha256", - ["methods", 0, "il_sha256"], - "0".repeat(64), - /IL sha256 mismatch/u, - ], - [ - "normalized IL sha256", - ["methods", 0, "normalized_il_sha256"], - "0".repeat(64), - /normalized IL sha256 mismatch/u, - ], + { + label: "target sha256", + owner: ["target"], + key: "sha256", + wrong: "0".repeat(64), + expected: /target sha256 mismatch/u, + }, + { + label: "method token", + owner: ["methods", 0], + key: "token", + wrong: "0x0600ffff", + expected: /method 0x0600ffff was not returned/u, + }, + { + label: "signature sha256", + owner: ["methods", 0], + key: "signature_sha256", + wrong: "0".repeat(64), + expected: /signature sha256 mismatch/u, + }, + { + label: "IL size", + owner: ["methods", 0], + key: "il_size", + wrong: 999, + expected: /IL size mismatch/u, + }, + { + label: "IL sha256", + owner: ["methods", 0], + key: "il_sha256", + wrong: "0".repeat(64), + expected: /IL sha256 mismatch/u, + }, + { + label: "normalized IL sha256", + owner: ["methods", 0], + key: "normalized_il_sha256", + wrong: "0".repeat(64), + expected: /normalized IL sha256 mismatch/u, + }, ]; - for (const [label, path, wrong, expected] of refusals) { + for (const { label, owner: ownerPath, key, wrong, expected } of refusals) { const rejected = structuredClone(manifest); - let owner = rejected; - for (const key of path.slice(0, -1)) owner = owner[key]; - const key = path.at(-1); + const owner = manifestField(rejected, ownerPath); assert.notEqual( owner[key], wrong, @@ -145,9 +192,34 @@ async function verifyManifestRejections(context, manifest, manifestPath) { `Manifest verifier accepted an authored wrong ${label}`, ); } - return refusals.map(([label]) => label); + return refusals.map(({ label }) => label); } +/** + * The object or array at `path` inside a manifest. + * @param {unknown} manifest + * @param {readonly (string | number)[]} path + */ +function manifestField(manifest, path) { + let field = manifest; + for (const key of path) field = manifestObject(field, path)[key]; + return manifestObject(field, path); +} + +/** + * @param {unknown} value + * @param {readonly (string | number)[]} path + * @returns {Record} + */ +function manifestObject(value, path) { + assert.ok( + typeof value === "object" && value !== null, + `manifest has no object at ${path.join(".")}`, + ); + return /** @type {Record} */ (value); +} + +/** @param {ManagedOracleSupport} context */ async function runOptionalManagedAppManifest(context) { const manifestPath = process.env.REA_MANAGED_APP_MANIFEST_PATH; if (manifestPath === undefined) return null; @@ -170,6 +242,7 @@ async function runOptionalManagedAppManifest(context) { }; } +/** @param {ManagedOracleSupport} context */ async function runOptionalIlspyOracle(context) { const ilspyPath = process.env.REA_ILSPY_CMD_PATH; if (ilspyPath === undefined) return null; @@ -214,7 +287,7 @@ async function runOptionalIlspyOracle(context) { csharp.length <= 65_536, "ilspycmd C# output exceeded the managed reconstruction import bound", ); - const reconstructionImport = importIlspyReconstruction({ + const reconstruction = importIlspyReconstruction({ sample, method, members, @@ -222,7 +295,9 @@ async function runOptionalIlspyOracle(context) { executableSha256, csharp, }); - const imported = reconstructionImport.value.normalized_result; + const imported = managedReconstructionImportResultSchema.parse( + reconstruction.normalized_result, + ); return { env: "REA_ILSPY_CMD_PATH", configured: true, @@ -246,7 +321,7 @@ async function runOptionalIlspyOracle(context) { class_listing_sha256: context.sha256(Buffer.from(listOutput)), decompiled_csharp_sha256: context.sha256(Buffer.from(csharp)), decompiled_csharp_bytes: Buffer.byteLength(csharp), - imported_evidence_id: reconstructionImport.value.evidence_id, + imported_evidence_id: reconstruction.evidence_id, reconstruction_id: imported.reconstruction_id, canonical_observation: imported.methods[0]?.validation.canonical_observation ?? null, @@ -256,6 +331,18 @@ async function runOptionalIlspyOracle(context) { }; } +/** + * Import the ilspycmd output as managed reconstruction Evidence. + * @param {{ + * readonly sample: ManagedOracleFixture; + * readonly method: ManagedMethod; + * readonly members: ManagedMemberInspection; + * readonly version: string; + * readonly executableSha256: string; + * readonly csharp: string; + * }} input + * @returns {Evidence} + */ function importIlspyReconstruction({ sample, method, @@ -302,14 +389,17 @@ function importIlspyReconstruction({ ], }); ensureIlspy(reconstructionImport.ok, "managed reconstruction import failed"); - const imported = reconstructionImport.value.normalized_result; + const imported = managedReconstructionImportResultSchema.parse( + reconstructionImport.value.normalized_result, + ); ensureIlspy( imported.summary.decompiled_csharp_methods === 1, "managed reconstruction import did not record one decompiled C# method", ); - return reconstructionImport; + return reconstructionImport.value; } +/** @param {string} path */ async function ensureExecutable(path) { try { await access(path, constants.X_OK); @@ -318,6 +408,11 @@ async function ensureExecutable(path) { } } +/** + * @param {string} path + * @param {readonly string[]} args + * @param {number} maxBuffer + */ async function runIlspy(path, args, maxBuffer) { try { const { stdout } = await execFileAsync(path, args, { @@ -335,6 +430,7 @@ async function runIlspy(path, args, maxBuffer) { } } +/** @param {string} output */ function ilspyVersion(output) { const version = output .split(/\r?\n/u) @@ -347,6 +443,11 @@ function ilspyVersion(output) { return version; } +/** + * @param {boolean} condition + * @param {string} message + * @returns {asserts condition} + */ function ensureIlspy(condition, message) { if (!condition) throw new Error(`managed ILSpy oracle failed: ${message}`); } diff --git a/tests/boundary/process/mcpVerifierResults.test.ts b/tests/boundary/process/mcpVerifierResults.test.ts index 5e65712dc1..d553953a10 100644 --- a/tests/boundary/process/mcpVerifierResults.test.ts +++ b/tests/boundary/process/mcpVerifierResults.test.ts @@ -44,6 +44,19 @@ interface VerifierResults { readonly expandProviderOutput: (providers: unknown) => { readonly analysis_provider_candidates: unknown; }; + readonly requireEvidenceProvider: ( + result: unknown, + operation: string, + providerId: string, + ) => void; + readonly requireWorkflowEvidenceProvider: ( + result: unknown, + operation: string, + expected: { + readonly workflowProviderId: string; + readonly upstreamProviderId: string; + }, + ) => void; } // The verifier helper imports the compiled contracts, so this runs after build. @@ -435,6 +448,77 @@ it("requires canonical complete tool errors before inspecting their diagnostic f ); }); +it("requires concrete and composed provider provenance in Evidence text", async () => { + const { requireEvidenceProvider, requireWorkflowEvidenceProvider } = + await verifierResults(); + const versioned = { id: "hopper", version: "5.0" }; + const concrete = { + provider: { id: "hopper" }, + analysis_profile: { provider: versioned }, + }; + expect(() => + requireEvidenceProvider(textReply(concrete), "list_procedures", "hopper"), + ).not.toThrow(); + for (const value of [ + null, + { ...concrete, provider: { id: "ghidra" } }, + { ...concrete, analysis_profile: { provider: { id: "hopper" } } }, + { + ...concrete, + analysis_profile: { provider: { ...versioned, version: null } }, + }, + ]) + expect(() => + requireEvidenceProvider(textReply(value), "list_procedures", "hopper"), + ).toThrow("list_procedures omitted its concrete provider provenance"); + const workflow = { + provider: { id: "rea-workflow" }, + analysis_profile: { + provider: { id: "rea-workflow" }, + parameters: { upstream_analysis_profile: { provider: versioned } }, + }, + }; + const expected = { + workflowProviderId: "rea-workflow", + upstreamProviderId: "hopper", + }; + expect(() => + requireWorkflowEvidenceProvider( + textReply(workflow), + "binary_overview", + expected, + ), + ).not.toThrow(); + for (const value of [ + concrete, + { + ...workflow, + analysis_profile: { + ...workflow.analysis_profile, + parameters: { + upstream_analysis_profile: { provider: { id: "hopper" } }, + }, + }, + }, + { + ...workflow, + analysis_profile: { + ...workflow.analysis_profile, + provider: { id: "hopper" }, + }, + }, + ]) + expect(() => + requireWorkflowEvidenceProvider( + textReply(value), + "binary_overview", + expected, + ), + ).toThrow( + "binary_overview omitted its composed workflow or upstream provenance", + ); +}); + it.each(["transport disagreement", "malformed normalized result"])( "admits complete Wasm Evidence at the interface verifier boundary: %s", async (scenario) => { diff --git a/tsconfig.scripts.json b/tsconfig.scripts.json new file mode 100644 index 0000000000..607513dc93 --- /dev/null +++ b/tsconfig.scripts.json @@ -0,0 +1,24 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "noEmit": true, + "paths": { + "#dist/*": ["./src/*"] + } + }, + "files": [ + "scripts/lib/mcp-verifier-results.mjs", + "scripts/lib/real-ghidra-snapshot-lifecycle.mjs", + "scripts/lib/verifier-artifact-scope.mjs", + "scripts/verify-package-evidence.mjs", + "scripts/verify/javascript/digests.mjs", + "scripts/verify/javascript/output.mjs", + "scripts/verify/managed/completion-report.mjs", + "scripts/verify/managed/manifest-parser.mjs", + "scripts/verify/managed/manifest.mjs", + "scripts/verify/managed/oracles.mjs" + ], + "include": [] +} diff --git a/turbo.json b/turbo.json index 8c68b4fe12..3e96046d7d 100644 --- a/turbo.json +++ b/turbo.json @@ -46,8 +46,10 @@ "tests/**/package.json", "tsconfig.json", "tsconfig.build.json", + "tsconfig.scripts.json", "vitest.config.ts", "scripts/**/*.d.{cts,mts,ts}", + "scripts/**/*.mjs", "scripts/**/package.json", ".github/actions/**/*.{cts,mts,ts,tsx}", ".github/**/package.json",