build(scripts): type-check the Evidence verification scripts against src (#11579)

* build(scripts): type-check the Evidence verification scripts against src

Six JavaScript callers kept passing `undefined` as an Evidence subject
after `createEvidence` stopped accepting it (#11536); `npm run typecheck`
saw only the TypeScript callers. The scripts import built modules by
relative `../dist/` paths, which a checker resolves to the built JavaScript
(or nothing, before a build) rather than the source types.

A package-private `#dist/*` import alias resolves to `./dist/*` at run
time, and `tsconfig.scripts.json` maps it to `./src/*` and checks the
listed scripts with `checkJs` under the main compiler options. `typecheck`,
and so `check:fast`, runs it; turbo's typecheck inputs include the scripts
and the new config.

First slice: `verify-package-evidence.mjs` (its evidence-bundle import
becomes static), `verify/javascript/output.mjs` and
`verify/javascript/digests.mjs`, with the JSDoc types the check required.

Refs #11545

* build(scripts): type-check the MCP verifier results and Ghidra snapshot lifecycle helpers (#11590)

* build(scripts): type-check the managed app manifest verifier (#11592)

* build(scripts): type-check the managed oracles and completion report (#11605)

oracles.mjs and completion-report.mjs join tsconfig.scripts.json (72 errors). ensureIlspy is an assertion function, the ILSpy import returns the imported Evidence and reads its normalized result through managedReconstructionImportResultSchema, the manifest-rejection controls are named records walked by a checked field lookup, and the completion report names its input.
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 08:04:12 +08:00
1 parent 6eb42a8d8d
commit b3ca03972a
15 files changed
+847 -169

No files matched your search

+14
View File
@@ -48,6 +48,20 @@ that invoke PowerShell themselves remain subject to their own policy checks.
| `npm run docs:check` | Generated-document validation and fresh onboarding example consistency checks |
| `npm run docs:build` | Render VitePress documentation and apply its page-link checks; no live external-link or GitHub fragment verification |
`typecheck` also checks the JavaScript verification scripts listed in
`tsconfig.scripts.json` (`checkJs`, with the main compiler options). Those
scripts import built modules through the package-private `#dist/*` alias, which
Node resolves to `./dist/*` at run time and the check resolves to `./src/*`, so a
domain signature change that breaks a listed script fails `check:fast` without a
build. To add a script, switch its `../dist/` imports and those of the local
modules it loads to `#dist/`, add it to the list, and fix what the check
reports with JSDoc types. Name tool results with the
`scripts/lib/mcp-verifier-results.mjs` typedefs (`ToolName`,
`VerifierOperationResult<N>`) so a result reads with its named contract's
type. `tests/support/analysis-efficiency/android-mapping-retention.mjs` stays
outside the check: it imports `dist` modules by path from a caller-selected
checkout at run time, so no import in this tree names their types.
The general source suite above runs on Linux in CI, and Windows CI runs a
curated lane ([Native platform baseline in CI](#native-platform-baseline-in-ci)).
A Windows source checkout can fail source tests for reasons unrelated to the
+2 -1
View File
@@ -35,6 +35,7 @@
},
"type": "module",
"imports": {
"#dist/*": "./dist/*",
"#javascript-analysis-worker": "./dist/composition/javascriptAnalysisWorkerProcess.js",
"#web-bundle-analysis-worker": "./dist/composition/webBundleAnalysisWorkerProcess.js"
},
@@ -192,7 +193,7 @@
"test:watch": "vitest --project domain --project services --project adapters --project composition --project conformance --project evaluation --changed",
"pretest:watch:all": "npm run test:prepare",
"test:watch:all": "vitest --changed",
"typecheck": "node scripts/run-typescript.mjs --noEmit",
"typecheck": "node scripts/run-typescript.mjs --noEmit && node scripts/run-typescript.mjs --noEmit -p tsconfig.scripts.json",
"verify:test-temp-hygiene": "npm run test:prepare && node scripts/verify-test-temp-hygiene.mjs",
"verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs",
"verify:hopper:callers": "npm run build:cached && node scripts/verify-real-hopper-callers.mjs",
+224 -54
View File
@@ -1,34 +1,98 @@
import { isDeepStrictEqual } from "node:util";
import { authenticateParsedEvidenceBundle } from "../../dist/domain/evidenceBundle.js";
import { analysisErrorProjectionSchema } from "../../dist/domain/analysisErrorProjectionSchema.js";
import { toolContract } from "../../dist/contracts/toolContracts.js";
import { z } from "zod";
import { authenticateParsedEvidenceBundle } from "#dist/domain/evidenceBundle.js";
import { analysisErrorProjectionSchema } from "#dist/domain/analysisErrorProjectionSchema.js";
import { toolContract } from "#dist/contracts/toolContracts.js";
import {
expandProviderCapabilities,
expandToolAvailability,
providerCapabilitySet,
} from "../../dist/contracts/toolOutputSchemaPrimitives.js";
import { compareUnicodeCodePoints } from "../../dist/domain/unicodeCodePointOrder.js";
} from "#dist/contracts/toolOutputSchemaPrimitives.js";
import { compareUnicodeCodePoints } from "#dist/domain/unicodeCodePointOrder.js";
/** Read the textual JSON projection from one MCP tool call. */
/** @typedef {import("@modelcontextprotocol/client").CallToolResult} CallToolResult */
/** @typedef {(typeof import("#dist/contracts/toolContracts.js").TOOL_CONTRACTS)[number]} PublicToolContract */
/** @typedef {PublicToolContract["name"]} ToolName */
/**
* @template {ToolName} N
* @typedef {Extract<PublicToolContract, { readonly name: N }>} NamedToolContract
*/
/**
* @template {PublicToolContract} C
* @typedef {z.output<C["outputSchema"]>} ToolOutput
*/
/**
* The member an MCP verifier projects: Evidence `normalized_result` or
* lifecycle `result`.
* @template {PublicToolContract} C
* @typedef {ToolOutput<C> extends { readonly normalized_result: infer R }
* ? R
* : ToolOutput<C> extends { readonly result: infer R }
* ? R
* : never} OperationResult
*/
/** @typedef {OperationResult<NamedToolContract<"binary_session">>} SessionStatus */
/**
* Session status as verifiers read it, with compact capability and tool
* groups expanded.
* @typedef {ReturnType<typeof expandMcpSessionStatus>} ExpandedSessionStatus
*/
/**
* @template {ToolName} N
* @typedef {N extends "binary_session"
* ? ExpandedSessionStatus
* : OperationResult<NamedToolContract<N>>} VerifierOperationResult
*/
/**
* Read the textual JSON projection from one MCP tool call.
* @param {CallToolResult} result
* @returns {string}
*/
export const mcpTextValue = (result) => {
const text = result.content?.find((item) => item.type === "text")?.text;
const text = result.content.find((item) => item.type === "text")?.text;
if (typeof text !== "string") throw new Error("Tool result omitted text");
return text;
};
/**
* @param {CallToolResult} result
* @returns {unknown}
*/
const jsonValue = (result) => JSON.parse(mcpTextValue(result));
/** Sort canonical contract names using REA's code-point ordering. */
/**
* Sort canonical contract names using REA's code-point ordering.
* @param {Iterable<string>} names
* @returns {string[]}
*/
export const sortVerifierNames = (names) =>
[...names].sort(compareUnicodeCodePoints);
/** Read the complete canonical diagnostic from an MCP tool failure. */
const toolErrorEnvelope = z.object({ error: z.unknown() });
/**
* Read the complete canonical diagnostic from an MCP tool failure.
* @param {CallToolResult} result
*/
export const requireMcpToolError = (result) => {
if (result.isError !== true) throw new Error("Expected an MCP tool failure");
return analysisErrorProjectionSchema.parse(jsonValue(result).error);
return analysisErrorProjectionSchema.parse(
toolErrorEnvelope.parse(jsonValue(result)).error,
);
};
/** Validate the complete output against its exact named contract, retaining inline Evidence. */
/**
* Validate the complete output against its exact named contract, retaining
* inline Evidence. Evidence outputs return the unprojected JSON value that
* satisfied the contract, so downstream parseEvidence authenticates source
* facts rather than a schema projection.
* @template {ToolName} N
* @param {CallToolResult} result
* @param {N} operation
* @param {NamedToolContract<N>} [contract]
* @returns {ToolOutput<NamedToolContract<N>>}
*/
export const requireMcpOutput = (
result,
operation,
@@ -39,23 +103,32 @@ export const requireMcpOutput = (
if (contract.name !== operation)
throw new Error(`${operation} received the ${contract.name} contract`);
const text = jsonValue(result);
const output = contract.outputSchema.parse(text);
// parse on a contract union widens to every contract's output; the guard
// above bound this contract to the operation's name.
const output = /** @type {ToolOutput<NamedToolContract<N>>} */ (
contract.outputSchema.parse(text)
);
if (result.structuredContent !== undefined) {
// The producer emits both representations from the same JSON value.
// Compare before schema projection can discard disagreeing source fields.
if (!isDeepStrictEqual(text, result.structuredContent))
throw new Error(`${operation} text and structured results disagree`);
}
// Downstream parseEvidence must authenticate source facts, not a schema projection.
return Object.hasOwn(output, "normalized_result") ? text : output;
return Object.hasOwn(output, "normalized_result")
? /** @type {ToolOutput<NamedToolContract<N>>} */ (text)
: output;
};
/**
* @param {PublicToolContract} contract
* @returns {"normalized_result" | "result"}
*/
const operationResultKey = (contract) => {
const shape = contract.outputSchema.shape;
if (shape === undefined)
const schema = contract.outputSchema;
if (!("shape" in schema))
throw new Error(`${contract.name} has an unsupported MCP output schema`);
const evidence = Object.hasOwn(shape, "normalized_result");
const lifecycle = Object.hasOwn(shape, "result");
const evidence = Object.hasOwn(schema.shape, "normalized_result");
const lifecycle = Object.hasOwn(schema.shape, "result");
if (evidence === lifecycle)
throw new Error(`${contract.name} has an unsupported MCP output schema`);
return evidence ? "normalized_result" : "result";
@@ -66,6 +139,7 @@ const operationResultKey = (contract) => {
* has validated the status against the binary_session contract, so a summary
* carries summary candidates without capability sets; complete detail must
* still expand every candidate's capabilities.
* @param {SessionStatus} status
*/
export const expandMcpSessionStatus = (status) => ({
...status,
@@ -77,7 +151,11 @@ export const expandMcpSessionStatus = (status) => ({
tool_availability: expandToolAvailability(status.tool_availability),
});
/** Expand provider capability groups while retaining candidate identity/state. */
/**
* Expand provider capability groups while retaining candidate identity/state.
* @template {{ readonly capabilities: unknown }} Candidate
* @param {readonly Candidate[]} candidates
*/
const expandProviderCandidates = (candidates) =>
candidates.map((candidate) => ({
...candidate,
@@ -86,7 +164,14 @@ const expandProviderCandidates = (candidates) =>
),
}));
/** Expand CLI provider capability groups at the same owning boundary. */
/**
* Expand CLI provider capability groups at the same owning boundary.
* @template {{
* readonly capabilities: unknown;
* readonly analysis_provider_candidates: readonly { readonly capabilities: unknown }[];
* }} Providers
* @param {Providers} providers
*/
export const expandProviderOutput = (providers) => ({
...providers,
capabilities: expandProviderCapabilities(
@@ -97,7 +182,11 @@ export const expandProviderOutput = (providers) => ({
),
});
/** Expand the CLI capabilities command's compact provider descriptor groups. */
/**
* Expand the CLI capabilities command's compact provider descriptor groups.
* @template {{ readonly capabilities: unknown }} Capabilities
* @param {Capabilities} capabilities
*/
export const expandCapabilitiesOutput = (capabilities) => ({
...capabilities,
capabilities: expandProviderCapabilities(
@@ -105,76 +194,157 @@ export const expandCapabilitiesOutput = (capabilities) => ({
),
});
/** Validate an Evidence-producing tool output and return its normalized result. */
/**
* Validate an Evidence-producing tool output and return its normalized result.
* @template {ToolName} N
* @param {CallToolResult} result
* @param {N} operation
* @returns {OperationResult<NamedToolContract<N>>}
*/
export const requireMcpEvidenceResult = (result, operation) => {
const contract = toolContract(operation);
if (operationResultKey(contract) !== "normalized_result")
throw new Error(`${operation} requires the lifecycle result extractor`);
return requireMcpOutput(result, operation, contract).normalized_result;
return projectOperationResult(
requireMcpOutput(result, operation, contract),
"normalized_result",
);
};
/** Validate a lifecycle tool output and return its operation result. */
/**
* Validate a lifecycle tool output and return its operation result.
* @template {ToolName} N
* @param {CallToolResult} result
* @param {N} operation
* @returns {VerifierOperationResult<N>}
*/
export const requireMcpLifecycleResult = (result, operation) => {
const contract = toolContract(operation);
if (operationResultKey(contract) !== "result")
throw new Error(`${operation} requires the Evidence result extractor`);
const projected = requireMcpOutput(result, operation, contract).result;
return operation === "binary_session"
? expandMcpSessionStatus(projected)
: projected;
return expandOperationResult(
operation,
projectOperationResult(
requireMcpOutput(result, operation, contract),
"result",
),
);
};
/** Require an authenticated complete bundle without changing summary-capable readers. */
/**
* Require an authenticated complete bundle without changing summary-capable readers.
* @param {CallToolResult} result
*/
export const requireMcpEvidenceBundle = (result) => {
const operation = "get_evidence_bundle";
const contract = toolContract(operation);
const bundle = requireMcpOutput(result, operation, contract).result;
if (bundle.kind === "evidence-bundle-summary")
const bundle = requireMcpOutput(result, "get_evidence_bundle").result;
if ("kind" in bundle)
throw new Error(
"get_evidence_bundle returned summary metadata instead of complete Evidence",
);
return authenticateParsedEvidenceBundle(bundle);
};
/** Project a mixed verifier call using its exact named canonical output schema. */
/**
* Project a mixed verifier call using its exact named canonical output schema.
* @template {ToolName} N
* @param {CallToolResult} result
* @param {N} operation
* @param {NamedToolContract<N>} [contract]
* @returns {VerifierOperationResult<N>}
*/
export const requireMcpOperationResult = (
result,
operation,
contract = toolContract(operation),
) => {
const key = operationResultKey(contract);
const projected = requireMcpOutput(result, operation, contract)[key];
return operation === "binary_session"
? expandMcpSessionStatus(projected)
: projected;
};
) =>
expandOperationResult(
operation,
projectOperationResult(
requireMcpOutput(result, operation, contract),
operationResultKey(contract),
),
);
/** Verify provider identity directly from the tool result. */
/**
* Read the member that operationResultKey selected from the contract.
* @template {PublicToolContract} C
* @param {ToolOutput<C>} output
* @param {"normalized_result" | "result"} key
* @returns {OperationResult<C>}
*/
const projectOperationResult = (output, key) =>
/** @type {Record<typeof key, OperationResult<C>>} */ (output)[key];
/**
* @template {ToolName} N
* @param {N} operation
* @param {OperationResult<NamedToolContract<N>>} projected
* @returns {VerifierOperationResult<N>}
*/
const expandOperationResult = (operation, projected) =>
/** @type {VerifierOperationResult<N>} */ (
operation === "binary_session"
? expandMcpSessionStatus(/** @type {SessionStatus} */ (projected))
: projected
);
const providerReference = z.object({ id: z.string() });
const versionedProviderReference = z.object({
id: z.string(),
version: z.string(),
});
const providerProvenance = z.object({
provider: providerReference,
analysis_profile: z.object({ provider: versionedProviderReference }),
});
const workflowProvenance = z.object({
provider: providerReference,
analysis_profile: z.object({
provider: providerReference,
parameters: z.object({
upstream_analysis_profile: z.object({
provider: versionedProviderReference,
}),
}),
}),
});
/**
* Verify provider identity directly from the tool result.
* @param {CallToolResult} result
* @param {string} operation
* @param {string} providerId
*/
export const requireEvidenceProvider = (result, operation, providerId) => {
const evidence = jsonValue(result);
const evidence = providerProvenance.safeParse(jsonValue(result));
if (
evidence?.provider?.id !== providerId ||
evidence?.analysis_profile?.provider?.id !== providerId ||
typeof evidence.analysis_profile.provider.version !== "string"
!evidence.success ||
evidence.data.provider.id !== providerId ||
evidence.data.analysis_profile.provider.id !== providerId
) {
throw new Error(`${operation} omitted its concrete provider provenance`);
}
};
/** Verify composed workflow and upstream provider provenance inline. */
/**
* Verify composed workflow and upstream provider provenance inline.
* @param {CallToolResult} result
* @param {string} operation
* @param {{ readonly workflowProviderId: string; readonly upstreamProviderId: string }} expected
*/
export const requireWorkflowEvidenceProvider = (
result,
operation,
expected,
) => {
const evidence = jsonValue(result);
const profile = evidence?.analysis_profile;
const upstream = profile?.parameters?.upstream_analysis_profile;
const evidence = workflowProvenance.safeParse(jsonValue(result));
if (
evidence?.provider?.id !== expected.workflowProviderId ||
profile?.provider?.id !== expected.workflowProviderId ||
upstream?.provider?.id !== expected.upstreamProviderId ||
typeof upstream.provider.version !== "string"
!evidence.success ||
evidence.data.provider.id !== expected.workflowProviderId ||
evidence.data.analysis_profile.provider.id !==
expected.workflowProviderId ||
evidence.data.analysis_profile.parameters.upstream_analysis_profile.provider
.id !== expected.upstreamProviderId
) {
throw new Error(
`${operation} omitted its composed workflow or upstream provenance`,
+94 -26
View File
@@ -6,10 +6,58 @@ import assert from "node:assert/strict";
import { access, mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { parseAnalysisSnapshot } from "../../dist/domain/analysisSnapshot.js";
import { parseEvidenceBundle } from "../../dist/domain/evidenceBundle.js";
import { parseAnalysisSnapshot } from "#dist/domain/analysisSnapshot.js";
import { parseEvidenceBundle } from "#dist/domain/evidenceBundle.js";
import { toolContract } from "#dist/contracts/toolContracts.js";
/** Keep live Ghidra edits distinct from historical Evidence across snapshots. */
/** @typedef {import("@modelcontextprotocol/client").Client} Client */
/** @typedef {import("./mcp-verifier-results.mjs").ToolName} ToolName */
/**
* @template {ToolName} N
* @typedef {import("./mcp-verifier-results.mjs").VerifierOperationResult<N>} VerifierOperationResult
*/
/**
* The complete analyze_function dossier; a retained summary view has no
* procedure.
* @typedef {Extract<VerifierOperationResult<"analyze_function">, { readonly procedure: unknown }>} FunctionDossier
*/
/**
* Call one tool and return its contract-validated operation result.
* @typedef {<N extends ToolName>(name: N, args?: Record<string, unknown>) => Promise<VerifierOperationResult<N>>} CallTool
*/
/**
* Require a snapshot binding rejection from one tool call.
* @typedef {(name: ToolName, args: Record<string, unknown>) => Promise<void>} RejectSnapshot
*/
/**
* Require the complete dossier that `detail: "complete"` selects.
* @param {VerifierOperationResult<"analyze_function">} result
* @returns {FunctionDossier}
*/
const completeDossier = (result) => {
assert.ok(
"procedure" in result,
"analyze_function returned a summary view instead of its dossier",
);
return result;
};
// Bundle records keep each operation's normalized result as JSON; read it
// through that operation's own contract.
const annotationResult = toolContract("annotate_native_function").outputSchema
.shape.normalized_result;
const nativeApiResult =
toolContract("inspect_native_api").outputSchema.shape.normalized_result;
/**
* Keep live Ghidra edits distinct from historical Evidence across snapshots.
* @param {Client} client
* @param {{ readonly path: string; readonly sha256: string }} target
* @param {FunctionDossier} edited The MCP session's dossier after its edit.
* @param {(command: "function", value: string, flags?: readonly string[]) => Promise<FunctionDossier>} cli
* Runs a one-shot CLI command against its own unmodified database.
*/
export async function verifyGhidraSnapshotLifecycle(
client,
target,
@@ -19,6 +67,7 @@ export async function verifyGhidraSnapshotLifecycle(
const workspace = await mkdtemp(join(tmpdir(), "rea-ghidra-snapshot-proof-"));
let successfulCalls = 0;
let rejectedCalls = 0;
/** @type {CallTool} */
const call = async (name, args = {}) => {
const result = requireMcpOperationResult(
await client.callTool({ name, arguments: args }, { timeout: 180000 }),
@@ -27,6 +76,7 @@ export async function verifyGhidraSnapshotLifecycle(
successfulCalls++;
return result;
};
/** @type {RejectSnapshot} */
const rejectSnapshot = async (name, args) => {
const reply = await client.callTool({ name, arguments: args });
assert.equal(
@@ -43,6 +93,11 @@ export async function verifyGhidraSnapshotLifecycle(
assert.equal(error.retryable, false);
rejectedCalls++;
};
const currentRunId = async () => {
const { analysis_run: run } = await call("binary_session");
assert.ok(run !== null, "binary_session omitted the open analysis run");
return run.run_id;
};
try {
const entry = edited.procedure.address;
const snapshotPath = join(workspace, "original.json");
@@ -54,7 +109,7 @@ export async function verifyGhidraSnapshotLifecycle(
);
assert.equal(snapshot.target.identity.sha256, target.sha256);
assert.notEqual(original.procedure.name, edited.procedure.name);
const runId = (await call("binary_session")).analysis_run.run_id;
const runId = await currentRunId();
const api = await call("inspect_native_api", { procedure: entry });
assert.equal(api.procedure.name, edited.procedure.name);
for (const repeatedOpen of [false, true]) {
@@ -66,7 +121,7 @@ export async function verifyGhidraSnapshotLifecycle(
);
await rejectSnapshot("close_binary", { snapshot_path: destination });
await assert.rejects(access(destination), { code: "ENOENT" });
assert.equal((await call("binary_session")).analysis_run.run_id, runId);
assert.equal(await currentRunId(), runId);
assert.deepEqual(
await call("analyze_function", {
procedure: entry,
@@ -81,7 +136,7 @@ export async function verifyGhidraSnapshotLifecycle(
snapshot_path: snapshotPath,
});
assert.deepEqual(await readFile(snapshotPath), snapshotBytes);
assert.equal((await call("binary_session")).analysis_run.run_id, runId);
assert.equal(await currentRunId(), runId);
assert.deepEqual(
await call("analyze_function", { procedure: entry, detail: "complete" }),
edited,
@@ -93,19 +148,22 @@ export async function verifyGhidraSnapshotLifecycle(
const bundle = parseEvidenceBundle(
JSON.parse(await readFile(bundlePath, "utf8")),
);
const annotation = bundle.records.find(
(record) =>
record.operation === "annotate_native_function" &&
record.normalized_result.annotations.name === edited.procedure.name &&
record.normalized_result.annotations.address ===
edited.procedure.address,
);
const editedApi = bundle.records.find(
(record) =>
record.operation === "inspect_native_api" &&
record.normalized_result.procedure.name === edited.procedure.name &&
record.normalized_result.procedure.address === edited.procedure.address,
);
const annotation = bundle.records.find((record) => {
if (record.operation !== "annotate_native_function") return false;
const { annotations } = annotationResult.parse(record.normalized_result);
return (
annotations.name === edited.procedure.name &&
annotations.address === edited.procedure.address
);
});
const editedApi = bundle.records.find((record) => {
if (record.operation !== "inspect_native_api") return false;
const { procedure } = nativeApiResult.parse(record.normalized_result);
return (
procedure.name === edited.procedure.name &&
procedure.address === edited.procedure.address
);
});
assert.ok(annotation);
assert.ok(editedApi);
@@ -115,13 +173,12 @@ export async function verifyGhidraSnapshotLifecycle(
provider_id: "ghidra",
snapshot_path: snapshotPath,
});
const restored = await call("analyze_function", {
procedure: entry,
detail: "complete",
});
const restored = completeDossier(
await call("analyze_function", { procedure: entry, detail: "complete" }),
);
assert.deepEqual(restored.procedure, original.procedure);
assert.deepEqual(restored.comments, original.comments);
assert.notEqual((await call("binary_session")).analysis_run.run_id, runId);
assert.notEqual(await currentRunId(), runId);
const freshPath = join(workspace, "fresh.json");
await call("close_binary", { snapshot_path: freshPath });
const fresh = parseAnalysisSnapshot(
@@ -153,6 +210,16 @@ export async function verifyGhidraSnapshotLifecycle(
}
}
/**
* @param {{
* readonly call: CallTool;
* readonly rejectSnapshot: RejectSnapshot;
* readonly workspace: string;
* readonly target: { readonly path: string };
* readonly entry: string;
* readonly original: FunctionDossier;
* }} context
*/
async function verifyConcurrentSnapshotClose({
call,
rejectSnapshot,
@@ -180,8 +247,9 @@ async function verifyConcurrentSnapshotClose({
await call("close_binary");
await call("open_binary", { path: target.path, provider_id: "ghidra" });
assert.deepEqual(
(await call("analyze_function", { procedure: entry, detail: "complete" }))
.procedure,
completeDossier(
await call("analyze_function", { procedure: entry, detail: "complete" }),
).procedure,
original.procedure,
);
}
+13 -2
View File
@@ -1,7 +1,12 @@
import { ArtifactResourceScope } from "../../dist/resources/ArtifactResourceScope.js";
import { ArtifactResourceScope } from "#dist/resources/ArtifactResourceScope.js";
/** A failed close retains the scope that can retry all of its owners. */
export class VerifierArtifactCleanupFailure extends AggregateError {
/**
* @param {ArtifactResourceScope} scope
* @param {readonly unknown[]} failures
* @param {unknown} cause
*/
constructor(scope, failures, cause) {
super(failures, "Verifier cleanup failed", { cause });
this._tag = "VerifierArtifactCleanupFailure";
@@ -9,9 +14,15 @@ export class VerifierArtifactCleanupFailure extends AggregateError {
}
}
/** Run one verifier operation and retain its exact scope if close fails. */
/**
* Run one verifier operation and retain its exact scope if close fails.
* @template T
* @param {(scope: ArtifactResourceScope) => Promise<T>} operation
* @returns {Promise<T>}
*/
export async function withVerifierArtifactScope(operation) {
const scope = new ArtifactResourceScope();
/** @type {{ readonly ok: true; readonly value: T } | { readonly ok: false; readonly cause: unknown }} */
let outcome;
try {
outcome = { ok: true, value: await operation(scope) };
+22 -6
View File
@@ -1,10 +1,29 @@
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { createEvidence } from "../dist/domain/evidence.js";
import { unknownTarget } from "../dist/domain/targetIdentity.js";
import { createEvidence } from "#dist/domain/evidence.js";
import {
createEvidenceBundle,
serializeEvidenceBundle,
} from "#dist/domain/evidenceBundle.js";
import { unknownTarget } from "#dist/domain/targetIdentity.js";
/** Round-trip historical reference import and evidence bundles. */
/**
* Round-trip historical reference import and evidence bundles.
* @param {{
* readonly commands: {
* run(
* command: string,
* arguments_: readonly string[],
* environment?: NodeJS.ProcessEnv,
* ): Promise<string>;
* };
* readonly cli: string;
* readonly evidenceRoot: string;
* readonly referenceRoot: string;
* readonly environment: NodeJS.ProcessEnv;
* }} options
*/
export async function verifyPackageEvidence({
commands,
cli,
@@ -26,9 +45,6 @@ export async function verifyPackageEvidence({
JSON.stringify(referenceImport).includes("PACKAGE_SECRET_SENTINEL")
)
throw new Error("packaged historical reference import CLI failed");
const { createEvidenceBundle, serializeEvidenceBundle } = await import(
new URL("../dist/domain/evidenceBundle.js", import.meta.url)
);
const sourceBundle = createEvidenceBundle([
createEvidence(
unknownTarget("not-observed"),
+3 -6
View File
@@ -4,12 +4,9 @@ import assert from "node:assert/strict";
import { constants } from "node:buffer";
import { createHash } from "node:crypto";
import { digestCanonicalValue } from "../../../dist/domain/canonicalDigest.js";
import {
createEvidence,
parseEvidence,
} from "../../../dist/domain/evidence.js";
import { unknownTarget } from "../../../dist/domain/targetIdentity.js";
import { digestCanonicalValue } from "#dist/domain/canonicalDigest.js";
import { createEvidence, parseEvidence } from "#dist/domain/evidence.js";
import { unknownTarget } from "#dist/domain/targetIdentity.js";
// This opt-in check crosses the actual engine string limit; keep it outside
// routine source tests. Build first, then run this script with the local Node.
+5 -4
View File
@@ -13,10 +13,10 @@ import { finished } from "node:stream/promises";
import { Cli } from "incur";
import { estimateTokenCount } from "tokenx";
import { parseCliOutputArguments } from "../../../dist/cliOutput.js";
import { createCliResultOutput } from "../../../dist/cli/cliResultOutput.js";
import { createEvidence } from "../../../dist/domain/evidence.js";
import { unknownTarget } from "../../../dist/domain/targetIdentity.js";
import { parseCliOutputArguments } from "#dist/cliOutput.js";
import { createCliResultOutput } from "#dist/cli/cliResultOutput.js";
import { createEvidence } from "#dist/domain/evidence.js";
import { unknownTarget } from "#dist/domain/targetIdentity.js";
import {
VerifierFixture,
withVerifierFixture,
@@ -39,6 +39,7 @@ const fixture = await VerifierFixture.open({
});
const workspace = fixture.path;
const path = join(workspace, "result.json");
/** @type {import("node:fs").WriteStream | undefined} */
let destination;
let peakRss = process.memoryUsage().rss;
const sampleRss = () => {
+50 -3
View File
@@ -1,5 +1,33 @@
import { createEvidence } from "../../../dist/domain/evidence.js";
import { unknownTarget } from "../../../dist/domain/targetIdentity.js";
import { createEvidence } from "#dist/domain/evidence.js";
import { unknownTarget } from "#dist/domain/targetIdentity.js";
/** @typedef {import("#dist/domain/binaryTargetTypes.js").BinaryTarget} BinaryTarget */
/** @typedef {import("#dist/domain/evidence.js").EvidenceSubjectTarget} EvidenceSubjectTarget */
/**
* A verified file as the manifest verifier and ILSpy oracle report it.
* @typedef {{ readonly file_name: string; readonly sha256: string }} ReportedTarget
*/
/**
* A source-owned fixture the conformance run wrote and resolved.
* @typedef {{ readonly target: BinaryTarget }} ResolvedFixture
*/
/**
* The completed conformance observations the claims are projected from.
* @typedef {{
* readonly modern: ResolvedFixture;
* readonly framework: ResolvedFixture;
* readonly nativeFunctionTarget: EvidenceSubjectTarget;
* readonly readyToRun: ResolvedFixture;
* readonly obfuscated: ResolvedFixture;
* readonly left: ResolvedFixture;
* readonly right: ResolvedFixture;
* readonly nativeOnly: ResolvedFixture;
* readonly malformed: ResolvedFixture;
* readonly manifestSelfTest: { readonly target: ReportedTarget };
* readonly operatorManifest: { readonly target: ReportedTarget } | null;
* readonly ilspyOracle: { readonly fixture: ReportedTarget } | null;
* }} ManagedCompletionReportInput
*/
const COMPLETION_PROVIDER = {
id: "rea-managed-conformance",
@@ -7,6 +35,15 @@ const COMPLETION_PROVIDER = {
version: "1",
};
/**
* @param {{
* readonly claimId: string;
* readonly scenarioId: string;
* readonly targets?: readonly (BinaryTarget | EvidenceSubjectTarget)[];
* readonly status?: "pass" | "unsupported";
* readonly limitations?: readonly string[];
* }} claim
*/
const completionClaim = ({
claimId,
scenarioId,
@@ -48,11 +85,16 @@ const completionClaim = ({
};
};
/**
* @param {ReportedTarget | undefined} target
* @returns {EvidenceSubjectTarget[]}
*/
const optionalTarget = (target) =>
target === undefined
? []
: [{ path: target.file_name, sha256: target.sha256, format: "pe" }];
/** @param {ManagedCompletionReportInput} input */
const sourceOwnedClaims = ({
modern,
framework,
@@ -128,6 +170,7 @@ const sourceOwnedClaims = ({
}),
];
/** @param {ManagedCompletionReportInput} input */
const optionalClaims = ({ operatorManifest, ilspyOracle }) => [
completionClaim({
claimId: "managed.operator-local-manifest",
@@ -149,7 +192,11 @@ const optionalClaims = ({ operatorManifest, ilspyOracle }) => [
}),
];
/** Project completed managed assertions into the standard verifier report. */
/**
* Project completed managed assertions into the standard verifier report.
* @param {ManagedCompletionReportInput} input
* @param {unknown} verifierRun The completed run record, reported unchanged.
*/
export const createManagedCompletionReport = (input, verifierRun) => ({
verifier: {
id: COMPLETION_PROVIDER.id,
+74 -7
View File
@@ -1,8 +1,18 @@
import { isRecord } from "#dist/domain/isRecord.js";
/** @typedef {ReturnType<typeof parseManagedAppManifest>} ManagedAppManifest */
/** @typedef {ManagedAppManifest["methods"][number]} ManagedAppManifestMethod */
/** @typedef {NonNullable<ManagedAppManifest["application_graph"]>} ManagedAppManifestGraph */
/**
* Parse a caller-selected managed app manifest into its checked expectations.
* @param {unknown} rawManifest
*/
export function parseManagedAppManifest(rawManifest) {
const manifest = object(rawManifest, "manifest");
const target = object(manifest.target, "manifest.target");
const methods = array(manifest.methods, "manifest.methods");
ensure(manifest.methods.length > 0, "manifest.methods must not be empty");
ensure(methods.length > 0, "manifest.methods must not be empty");
return {
label: optionalString(manifest.label, "manifest.label"),
target: {
@@ -29,6 +39,7 @@ export function parseManagedAppManifest(rawManifest) {
};
}
/** @param {unknown} rawGraph */
function parseManagedAppManifestGraph(rawGraph) {
if (rawGraph === undefined) return undefined;
const graph = object(rawGraph, "manifest.application_graph");
@@ -61,6 +72,10 @@ function parseManagedAppManifestGraph(rawGraph) {
};
}
/**
* @param {unknown} rawTrace
* @param {number} index
*/
function parseManagedAppManifestTrace(rawTrace, index) {
const prefix = `manifest.application_graph.feature_traces[${String(index)}]`;
const trace = object(rawTrace, prefix);
@@ -76,7 +91,9 @@ function parseManagedAppManifestTrace(rawTrace, index) {
);
const minMatchedSeeds = trace.min_matched_seeds ?? 1;
ensure(
Number.isInteger(minMatchedSeeds) && minMatchedSeeds > 0,
typeof minMatchedSeeds === "number" &&
Number.isInteger(minMatchedSeeds) &&
minMatchedSeeds > 0,
`${prefix}.min_matched_seeds must be a positive integer`,
);
return {
@@ -89,6 +106,10 @@ function parseManagedAppManifestTrace(rawTrace, index) {
};
}
/**
* @param {unknown} rawMethod
* @param {number} index
*/
function parseManagedAppManifestMethod(rawMethod, index) {
const prefix = `manifest.methods[${String(index)}]`;
const method = object(rawMethod, prefix);
@@ -112,19 +133,31 @@ function parseManagedAppManifestMethod(rawMethod, index) {
};
}
/**
* @param {unknown} value
* @param {string} name
* @returns {{ readonly [key: string]: unknown }}
*/
function object(value, name) {
ensure(
value !== null && typeof value === "object" && !Array.isArray(value),
`${name} must be an object`,
);
ensure(isRecord(value), `${name} must be an object`);
return value;
}
/**
* @param {unknown} value
* @param {string} name
* @returns {readonly unknown[]}
*/
function array(value, name) {
ensure(Array.isArray(value), `${name} must be an array`);
return value;
}
/**
* @param {unknown} value
* @param {string} name
* @returns {string}
*/
function string(value, name) {
ensure(
typeof value === "string" && value.length > 0,
@@ -133,25 +166,42 @@ function string(value, name) {
return value;
}
/**
* @param {unknown} value
* @param {string} name
*/
function optionalString(value, name) {
if (value === undefined) return undefined;
return string(value, name);
}
/**
* @param {unknown} value
* @param {string} name
* @returns {number}
*/
function nonNegativeInteger(value, name) {
ensure(
Number.isInteger(value) && value >= 0,
typeof value === "number" && Number.isInteger(value) && value >= 0,
`${name} must be a non-negative integer`,
);
return value;
}
/**
* @param {unknown} value
* @param {string} name
*/
function digest(value, name) {
const text = string(value, name);
ensure(/^[a-f0-9]{64}$/u.test(text), `${name} must be a lowercase sha256`);
return text;
}
/**
* @param {unknown} value
* @param {string} name
*/
function optionalCliMetadataGuid(value, name) {
if (value === undefined) return undefined;
if (value === null) return null;
@@ -165,12 +215,20 @@ function optionalCliMetadataGuid(value, name) {
return text;
}
/**
* @param {unknown} value
* @param {string} name
*/
function metadataToken(value, name) {
const text = string(value, name);
ensure(/^0x[0-9a-f]{8}$/u.test(text), `${name} must be a metadata token`);
return text;
}
/**
* @param {unknown} value
* @param {string} name
*/
function methodToken(value, name) {
const token = metadataToken(value, name);
ensure(
@@ -180,12 +238,21 @@ function methodToken(value, name) {
return token;
}
/**
* The MethodDef row a checked method token names.
* @param {string} token
*/
export function methodTokenRow(token) {
const row = Number.parseInt(token.slice(4), 16);
ensure(row > 0, `method token ${token} has an invalid row id`);
return row;
}
/**
* @param {unknown} condition
* @param {string} message
* @returns {asserts condition}
*/
export function ensure(condition, message) {
if (!condition)
throw new Error(`managed app manifest verification failed: ${message}`);
+89 -14
View File
@@ -2,26 +2,60 @@ import { basename, dirname, isAbsolute, resolve } from "node:path";
import {
readManagedSnapshot,
verifyManagedSourceIdentity,
} from "../../../dist/dotnet/ManagedSnapshot.js";
} from "#dist/dotnet/ManagedSnapshot.js";
import { withVerifierArtifactScope } from "../../lib/verifier-artifact-scope.mjs";
import { parseBinaryTarget } from "../../../dist/application/BinaryTargetResolver.js";
import { inspectManagedArtifactBytes } from "../../../dist/dotnet/ManagedArtifactInspector.js";
import { inspectManagedMembersBytes } from "../../../dist/dotnet/ManagedMemberInspector.js";
import { projectManagedApplicationGraphEvidence } from "../../../dist/application/managed/ManagedApplicationGraphService.js";
import { traceApplicationFeatureEvidence } from "../../../dist/application/javascript/JavaScriptApplicationWorkflowService.js";
import { createEvidence } from "../../../dist/domain/evidence.js";
import { MANAGED_STATIC_PROVIDER } from "../../../dist/domain/investigationProviders.js";
import { parseBinaryTarget } from "#dist/application/BinaryTargetResolver.js";
import { inspectManagedArtifactBytes } from "#dist/dotnet/ManagedArtifactInspector.js";
import { inspectManagedMembersBytes } from "#dist/dotnet/ManagedMemberInspector.js";
import { projectManagedApplicationGraphEvidence } from "#dist/application/managed/ManagedApplicationGraphService.js";
import { traceApplicationFeatureEvidence } from "#dist/application/javascript/JavaScriptApplicationWorkflowService.js";
import { createEvidence } from "#dist/domain/evidence.js";
import { MANAGED_STATIC_PROVIDER } from "#dist/domain/investigationProviders.js";
import { managedApplicationGraphResultSchema } from "#dist/domain/managed/managedApplicationGraph.js";
import {
ensure,
methodTokenRow,
parseManagedAppManifest,
} from "./manifest-parser.mjs";
/** @typedef {import("#dist/domain/binaryTargetTypes.js").BinaryTarget} BinaryTarget */
/** @typedef {import("#dist/resources/ArtifactResourceScope.js").ArtifactResourceScope} ArtifactResourceScope */
/** @typedef {import("#dist/domain/evidence.js").Evidence} Evidence */
/** @typedef {ReturnType<typeof inspectManagedArtifactBytes>} ManagedArtifactInspection */
/** @typedef {ReturnType<typeof inspectManagedMembersBytes>["methods"][number]} ManagedMethod */
/** @typedef {import("./manifest-parser.mjs").ManagedAppManifest} ManagedAppManifest */
/** @typedef {import("./manifest-parser.mjs").ManagedAppManifestMethod} ManagedAppManifestMethod */
/** @typedef {import("./manifest-parser.mjs").ManagedAppManifestGraph} ManagedAppManifestGraph */
/**
* One manifest method's application graph projection.
* @typedef {{
* readonly method: ManagedMethod;
* readonly evidence: Evidence;
* readonly nodeKinds: readonly string[];
* readonly summary: import("zod").output<typeof managedApplicationGraphResultSchema>["summary"];
* }} ManifestMethodProjection
*/
/**
* Inputs shared by the graph projections of one verified target.
* @typedef {{
* readonly graphByToken: Map<string, ManifestMethodProjection>;
* readonly inspectedMethods: ReadonlyMap<string, ManagedMethod>;
* readonly bytes: Buffer;
* readonly target: BinaryTarget;
* readonly artifact: ManagedArtifactInspection;
* }} ProjectionState
*/
// Preserve fs.readFile admission: Node internal kIoMaxLength is 2 GiB minus one.
// https://nodejs.org/api/errors.html#err_fs_file_too_large
const READ_FILE_MAX_BYTES = 0x7fff_ffff;
/**
* Verify a caller-selected managed app manifest against its target.
* @param {unknown} rawManifest
* @param {string} manifestPath Resolves a relative target path.
*/
export async function verifyManagedAppManifest(rawManifest, manifestPath) {
return withVerifierArtifactScope((resources) =>
resources.run(() =>
@@ -30,6 +64,11 @@ export async function verifyManagedAppManifest(rawManifest, manifestPath) {
);
}
/**
* @param {unknown} rawManifest
* @param {string} manifestPath
* @param {ArtifactResourceScope} resources
*/
async function verifyManifestSnapshot(rawManifest, manifestPath, resources) {
const manifest = parseManagedAppManifest(rawManifest);
const targetPath = isAbsolute(manifest.target.path)
@@ -93,6 +132,10 @@ async function verifyManifestSnapshot(rawManifest, manifestPath, resources) {
};
}
/**
* @param {ManagedAppManifest} manifest
* @param {ManagedArtifactInspection} artifact
*/
function verifyManifestTarget(manifest, artifact) {
ensure(
artifact.classification.status === "managed",
@@ -121,9 +164,15 @@ function verifyManifestTarget(manifest, artifact) {
);
}
/**
* @param {ManagedAppManifest} manifest
* @param {Buffer} bytes
* @param {BinaryTarget} target
*/
function verifyManifestMethods(manifest, bytes, target) {
const methods = [];
let assertions = 0;
/** @type {Map<string, ManagedMethod>} */
const inspectedMethods = new Map();
for (const expectedMethod of manifest.methods) {
const method = inspectManagedMethodByToken(bytes, target, expectedMethod);
@@ -160,6 +209,16 @@ function verifyManifestMethods(manifest, bytes, target) {
return { methods, inspectedMethods, assertions };
}
/**
* @param {{
* readonly graphManifest: ManagedAppManifestGraph;
* readonly bytes: Buffer;
* readonly target: BinaryTarget;
* readonly artifact: ManagedArtifactInspection;
* readonly manifestMethods: readonly ManagedAppManifestMethod[];
* readonly inspectedMethods: ReadonlyMap<string, ManagedMethod>;
* }} options
*/
function verifyManagedAppManifestApplicationGraph({
graphManifest,
bytes,
@@ -168,6 +227,7 @@ function verifyManagedAppManifestApplicationGraph({
manifestMethods,
inspectedMethods,
}) {
/** @type {Map<string, ManifestMethodProjection>} */
const graphByToken = new Map();
const projectionState = {
graphByToken,
@@ -242,6 +302,11 @@ function verifyManagedAppManifestApplicationGraph({
};
}
/**
* @param {ProjectionState} state
* @param {string} token
* @returns {ManifestMethodProjection}
*/
const projectManifestMethod = (state, token) => {
const existing = state.graphByToken.get(token);
if (existing !== undefined) return existing;
@@ -286,20 +351,25 @@ const projectManifestMethod = (state, token) => {
projected.ok,
`application graph projection failed for method ${token}`,
);
// Read the projection through the managed graph owner's result schema.
const graph = managedApplicationGraphResultSchema.parse(
projected.value.normalized_result,
);
const projection = {
method,
evidence: projected.value,
nodeKinds: [
...new Set(
projected.value.normalized_result.graph.nodes.map(({ kind }) => kind),
),
].sort(),
summary: projected.value.normalized_result.summary,
nodeKinds: [...new Set(graph.graph.nodes.map(({ kind }) => kind))].sort(),
summary: graph.summary,
};
state.graphByToken.set(token, projection);
return projection;
};
/**
* @param {Buffer} bytes
* @param {BinaryTarget} target
* @param {ManagedAppManifestMethod} expectedMethod
*/
function inspectManagedMethodByToken(bytes, target, expectedMethod) {
const members = inspectManagedMethodPageByToken(
bytes,
@@ -320,6 +390,11 @@ function inspectManagedMethodByToken(bytes, target, expectedMethod) {
return method;
}
/**
* @param {Buffer} bytes
* @param {BinaryTarget} target
* @param {string} token
*/
function inspectManagedMethodPageByToken(bytes, target, token) {
const row = methodTokenRow(token);
return inspectManagedMembersBytes(bytes, target, {
+147 -46
View File
@@ -5,15 +5,46 @@ import { basename, isAbsolute, join, resolve } from "node:path";
import assert from "node:assert/strict";
import { promisify } from "node:util";
import { inspectManagedArtifactBytes } from "../../../dist/dotnet/ManagedArtifactInspector.js";
import { inspectManagedMembersBytes } from "../../../dist/dotnet/ManagedMemberInspector.js";
import { importManagedReconstructionEvidence } from "../../../dist/application/managed/ManagedReconstructionService.js";
import { createEvidence } from "../../../dist/domain/evidence.js";
import { MANAGED_STATIC_PROVIDER } from "../../../dist/domain/investigationProviders.js";
import { inspectManagedArtifactBytes } from "#dist/dotnet/ManagedArtifactInspector.js";
import { inspectManagedMembersBytes } from "#dist/dotnet/ManagedMemberInspector.js";
import { importManagedReconstructionEvidence } from "#dist/application/managed/ManagedReconstructionService.js";
import { createEvidence } from "#dist/domain/evidence.js";
import { MANAGED_STATIC_PROVIDER } from "#dist/domain/investigationProviders.js";
import { managedReconstructionImportResultSchema } from "#dist/domain/managed/managedReconstruction.js";
import { verifyManagedAppManifest } from "./manifest.mjs";
/** @typedef {import("#dist/domain/binaryTargetTypes.js").BinaryTarget} BinaryTarget */
/** @typedef {import("#dist/domain/evidence.js").Evidence} Evidence */
/** @typedef {ReturnType<typeof inspectManagedMembersBytes>} ManagedMemberInspection */
/** @typedef {ManagedMemberInspection["methods"][number]} ManagedMethod */
/**
* A source-owned managed fixture written into the verifier workspace.
* @typedef {{
* readonly bytes: Buffer;
* readonly path: string;
* readonly target: BinaryTarget;
* }} ManagedOracleFixture
*/
/**
* What the oracles need from the conformance support.
* @typedef {{
* readonly workspace: string;
* readonly defaultIlBody: Buffer;
* readonly fixture: (
* name: string,
* options: { readonly methodName: string; readonly ilBody?: Buffer },
* ) => Promise<ManagedOracleFixture>;
* readonly sha256: (bytes: Buffer) => string;
* }} ManagedOracleContext
*/
/** @typedef {ManagedOracleContext & { readonly verifyManagedAppManifest: typeof verifyManagedAppManifest }} ManagedOracleSupport */
const execFileAsync = promisify(execFile);
/** @param {ManagedOracleContext} context */
export const createManagedConformanceOracleSupport = (context) => {
const support = { ...context, verifyManagedAppManifest };
return {
@@ -23,6 +54,7 @@ export const createManagedConformanceOracleSupport = (context) => {
};
};
/** @param {ManagedOracleSupport} context */
async function runManagedAppManifestSelfTest(context) {
const sample = await context.fixture("manifest-self-test.exe", {
methodName: "PinnedSemanticSlice",
@@ -94,45 +126,60 @@ async function runManagedAppManifestSelfTest(context) {
};
}
/**
* Each authored mismatch must be refused by the manifest verifier.
* @param {ManagedOracleSupport} context
* @param {object} manifest A manifest the verifier accepts.
* @param {string} manifestPath
*/
async function verifyManifestRejections(context, manifest, manifestPath) {
const refusals = [
[
"target sha256",
["target", "sha256"],
"0".repeat(64),
/target sha256 mismatch/u,
],
[
"method token",
["methods", 0, "token"],
"0x0600ffff",
/method 0x0600ffff was not returned/u,
],
[
"signature sha256",
["methods", 0, "signature_sha256"],
"0".repeat(64),
/signature sha256 mismatch/u,
],
["IL size", ["methods", 0, "il_size"], 999, /IL size mismatch/u],
[
"IL sha256",
["methods", 0, "il_sha256"],
"0".repeat(64),
/IL sha256 mismatch/u,
],
[
"normalized IL sha256",
["methods", 0, "normalized_il_sha256"],
"0".repeat(64),
/normalized IL sha256 mismatch/u,
],
{
label: "target sha256",
owner: ["target"],
key: "sha256",
wrong: "0".repeat(64),
expected: /target sha256 mismatch/u,
},
{
label: "method token",
owner: ["methods", 0],
key: "token",
wrong: "0x0600ffff",
expected: /method 0x0600ffff was not returned/u,
},
{
label: "signature sha256",
owner: ["methods", 0],
key: "signature_sha256",
wrong: "0".repeat(64),
expected: /signature sha256 mismatch/u,
},
{
label: "IL size",
owner: ["methods", 0],
key: "il_size",
wrong: 999,
expected: /IL size mismatch/u,
},
{
label: "IL sha256",
owner: ["methods", 0],
key: "il_sha256",
wrong: "0".repeat(64),
expected: /IL sha256 mismatch/u,
},
{
label: "normalized IL sha256",
owner: ["methods", 0],
key: "normalized_il_sha256",
wrong: "0".repeat(64),
expected: /normalized IL sha256 mismatch/u,
},
];
for (const [label, path, wrong, expected] of refusals) {
for (const { label, owner: ownerPath, key, wrong, expected } of refusals) {
const rejected = structuredClone(manifest);
let owner = rejected;
for (const key of path.slice(0, -1)) owner = owner[key];
const key = path.at(-1);
const owner = manifestField(rejected, ownerPath);
assert.notEqual(
owner[key],
wrong,
@@ -145,9 +192,34 @@ async function verifyManifestRejections(context, manifest, manifestPath) {
`Manifest verifier accepted an authored wrong ${label}`,
);
}
return refusals.map(([label]) => label);
return refusals.map(({ label }) => label);
}
/**
* The object or array at `path` inside a manifest.
* @param {unknown} manifest
* @param {readonly (string | number)[]} path
*/
function manifestField(manifest, path) {
let field = manifest;
for (const key of path) field = manifestObject(field, path)[key];
return manifestObject(field, path);
}
/**
* @param {unknown} value
* @param {readonly (string | number)[]} path
* @returns {Record<string | number, unknown>}
*/
function manifestObject(value, path) {
assert.ok(
typeof value === "object" && value !== null,
`manifest has no object at ${path.join(".")}`,
);
return /** @type {Record<string | number, unknown>} */ (value);
}
/** @param {ManagedOracleSupport} context */
async function runOptionalManagedAppManifest(context) {
const manifestPath = process.env.REA_MANAGED_APP_MANIFEST_PATH;
if (manifestPath === undefined) return null;
@@ -170,6 +242,7 @@ async function runOptionalManagedAppManifest(context) {
};
}
/** @param {ManagedOracleSupport} context */
async function runOptionalIlspyOracle(context) {
const ilspyPath = process.env.REA_ILSPY_CMD_PATH;
if (ilspyPath === undefined) return null;
@@ -214,7 +287,7 @@ async function runOptionalIlspyOracle(context) {
csharp.length <= 65_536,
"ilspycmd C# output exceeded the managed reconstruction import bound",
);
const reconstructionImport = importIlspyReconstruction({
const reconstruction = importIlspyReconstruction({
sample,
method,
members,
@@ -222,7 +295,9 @@ async function runOptionalIlspyOracle(context) {
executableSha256,
csharp,
});
const imported = reconstructionImport.value.normalized_result;
const imported = managedReconstructionImportResultSchema.parse(
reconstruction.normalized_result,
);
return {
env: "REA_ILSPY_CMD_PATH",
configured: true,
@@ -246,7 +321,7 @@ async function runOptionalIlspyOracle(context) {
class_listing_sha256: context.sha256(Buffer.from(listOutput)),
decompiled_csharp_sha256: context.sha256(Buffer.from(csharp)),
decompiled_csharp_bytes: Buffer.byteLength(csharp),
imported_evidence_id: reconstructionImport.value.evidence_id,
imported_evidence_id: reconstruction.evidence_id,
reconstruction_id: imported.reconstruction_id,
canonical_observation:
imported.methods[0]?.validation.canonical_observation ?? null,
@@ -256,6 +331,18 @@ async function runOptionalIlspyOracle(context) {
};
}
/**
* Import the ilspycmd output as managed reconstruction Evidence.
* @param {{
* readonly sample: ManagedOracleFixture;
* readonly method: ManagedMethod;
* readonly members: ManagedMemberInspection;
* readonly version: string;
* readonly executableSha256: string;
* readonly csharp: string;
* }} input
* @returns {Evidence}
*/
function importIlspyReconstruction({
sample,
method,
@@ -302,14 +389,17 @@ function importIlspyReconstruction({
],
});
ensureIlspy(reconstructionImport.ok, "managed reconstruction import failed");
const imported = reconstructionImport.value.normalized_result;
const imported = managedReconstructionImportResultSchema.parse(
reconstructionImport.value.normalized_result,
);
ensureIlspy(
imported.summary.decompiled_csharp_methods === 1,
"managed reconstruction import did not record one decompiled C# method",
);
return reconstructionImport;
return reconstructionImport.value;
}
/** @param {string} path */
async function ensureExecutable(path) {
try {
await access(path, constants.X_OK);
@@ -318,6 +408,11 @@ async function ensureExecutable(path) {
}
}
/**
* @param {string} path
* @param {readonly string[]} args
* @param {number} maxBuffer
*/
async function runIlspy(path, args, maxBuffer) {
try {
const { stdout } = await execFileAsync(path, args, {
@@ -335,6 +430,7 @@ async function runIlspy(path, args, maxBuffer) {
}
}
/** @param {string} output */
function ilspyVersion(output) {
const version = output
.split(/\r?\n/u)
@@ -347,6 +443,11 @@ function ilspyVersion(output) {
return version;
}
/**
* @param {boolean} condition
* @param {string} message
* @returns {asserts condition}
*/
function ensureIlspy(condition, message) {
if (!condition) throw new Error(`managed ILSpy oracle failed: ${message}`);
}
@@ -44,6 +44,19 @@ interface VerifierResults {
readonly expandProviderOutput: (providers: unknown) => {
readonly analysis_provider_candidates: unknown;
};
readonly requireEvidenceProvider: (
result: unknown,
operation: string,
providerId: string,
) => void;
readonly requireWorkflowEvidenceProvider: (
result: unknown,
operation: string,
expected: {
readonly workflowProviderId: string;
readonly upstreamProviderId: string;
},
) => void;
}
// The verifier helper imports the compiled contracts, so this runs after build.
@@ -435,6 +448,77 @@ it("requires canonical complete tool errors before inspecting their diagnostic f
);
});
it("requires concrete and composed provider provenance in Evidence text", async () => {
const { requireEvidenceProvider, requireWorkflowEvidenceProvider } =
await verifierResults();
const versioned = { id: "hopper", version: "5.0" };
const concrete = {
provider: { id: "hopper" },
analysis_profile: { provider: versioned },
};
expect(() =>
requireEvidenceProvider(textReply(concrete), "list_procedures", "hopper"),
).not.toThrow();
for (const value of [
null,
{ ...concrete, provider: { id: "ghidra" } },
{ ...concrete, analysis_profile: { provider: { id: "hopper" } } },
{
...concrete,
analysis_profile: { provider: { ...versioned, version: null } },
},
])
expect(() =>
requireEvidenceProvider(textReply(value), "list_procedures", "hopper"),
).toThrow("list_procedures omitted its concrete provider provenance");
const workflow = {
provider: { id: "rea-workflow" },
analysis_profile: {
provider: { id: "rea-workflow" },
parameters: { upstream_analysis_profile: { provider: versioned } },
},
};
const expected = {
workflowProviderId: "rea-workflow",
upstreamProviderId: "hopper",
};
expect(() =>
requireWorkflowEvidenceProvider(
textReply(workflow),
"binary_overview",
expected,
),
).not.toThrow();
for (const value of [
concrete,
{
...workflow,
analysis_profile: {
...workflow.analysis_profile,
parameters: {
upstream_analysis_profile: { provider: { id: "hopper" } },
},
},
},
{
...workflow,
analysis_profile: {
...workflow.analysis_profile,
provider: { id: "hopper" },
},
},
])
expect(() =>
requireWorkflowEvidenceProvider(
textReply(value),
"binary_overview",
expected,
),
).toThrow(
"binary_overview omitted its composed workflow or upstream provenance",
);
});
it.each(["transport disagreement", "malformed normalized result"])(
"admits complete Wasm Evidence at the interface verifier boundary: %s",
async (scenario) => {
+24
View File
@@ -0,0 +1,24 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"allowJs": true,
"checkJs": true,
"noEmit": true,
"paths": {
"#dist/*": ["./src/*"]
}
},
"files": [
"scripts/lib/mcp-verifier-results.mjs",
"scripts/lib/real-ghidra-snapshot-lifecycle.mjs",
"scripts/lib/verifier-artifact-scope.mjs",
"scripts/verify-package-evidence.mjs",
"scripts/verify/javascript/digests.mjs",
"scripts/verify/javascript/output.mjs",
"scripts/verify/managed/completion-report.mjs",
"scripts/verify/managed/manifest-parser.mjs",
"scripts/verify/managed/manifest.mjs",
"scripts/verify/managed/oracles.mjs"
],
"include": []
}
+2
View File
@@ -46,8 +46,10 @@
"tests/**/package.json",
"tsconfig.json",
"tsconfig.build.json",
"tsconfig.scripts.json",
"vitest.config.ts",
"scripts/**/*.d.{cts,mts,ts}",
"scripts/**/*.mjs",
"scripts/**/package.json",
".github/actions/**/*.{cts,mts,ts,tsx}",
".github/**/package.json",