mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 06:04:03 +00:00
* test(javascript): cover the isolated-world probe outside Electron identity
* fix(javascript): expand Electron call paths only with an Electron reference
* test(javascript): cover Electron reach through resolved names, aliases and worker URLs
* fix(javascript): attribute call-path unknowns only where the walk can reach Electron
* test(javascript): cover a hoisted read that precedes a loader alias
* fix(javascript): keep call-path unknowns for late-initialized loader aliases
* test(javascript): drop a duplicate loader-alias case
* test(javascript): cover loader aliases reached through bindings and calls
* fix(javascript): prove a late initializer loader-free before dropping its unknown
* test(javascript): cover loaders behind default parameters, wrappers and long alias chains
* fix(javascript): resolve identifiers inside late initializers and bound the loader proof
* fix(javascript): bound late loader proofs and preserve input uncertainty
* fix(javascript): fail closed for receiver and mutated loaders
* fix(javascript): close loader-free proof gaps
* fix(javascript): retain uncertain loader sources
* test(javascript): cover remaining loader proof gaps
* fix(javascript): bound alias writes and preserve loader uncertainty
* test(javascript): cover demand-driven loader mutation facts
* perf(javascript): index loader mutation facts on demand
* style(javascript): apply oxfmt to the loader proof changes
* test(javascript): cover inherited loaders and rebound alias writes
* fix(javascript): preserve inherited loaders and rebound aliases
* test(evaluation): update scoped feature coverage oracle
* test(javascript): cover inherited loader proof gaps
* fix(javascript): preserve projected and inherited loaders
* fix(javascript): prove safe parameter member reads
* test(javascript): cover loader proof review gaps
* fix(javascript): close remaining loader proof gaps
* fix(javascript): preserve qualified global replacements
* fix(javascript): preserve spread rest uncertainty
* fix(javascript): retain retried handler uncertainty
* fix(javascript): retain import.meta loader uncertainty
* fix(javascript): detach fresh constructed aliases
* fix(javascript): check inherited members of ambient loaders
* test(javascript): parse call-path gate fixtures as established CommonJS
* fix(javascript): prove projected locals inside late initializers
A function-local binding was accepted by scope alone, so a destructuring
projection such as `const { r } = {}` inside a late initializer never met
the inherited-member check that a top-level projection goes through. Route
projected locals through the binding proof; locals with inline initializers
keep the scope shortcut, since the walk already visits their expressions.
* fix(javascript): keep late results with changed prototypes or import.meta uncertain
The loader-free proof inspected member expressions inside a late function
but not what its caller does with the result. With `Object.prototype.load =
require`, `var load = () => ({})` and `load().load("electron")` reach the
loader through a member selected after the call, and a bare `import.meta`
returned after `import.meta.load = ...` does the same. Check each result
expression: a literal fails when its prototype chain gained members, a bare
`import.meta` when any of its members changed, and a result of unknown shape
when either happened. Promise and generator results inherit from
Object.prototype as well. `SemanticPropertyPathCoverage.touches` answers
whether any retained write or escape sits below a prefix.
* fix(javascript): detach replacements held by fresh bindings
`alias = replacement` with `const replacement = {}` selected an identifier
initializer, so the later `alias.other = 1` stayed a shared write on the
original alias group and the resolver treated `base` as overwritten,
dropping its bridge call path. Resolve the identifier to its binding and
accept it when that binding only ever received one fresh allocation.
Bindings that can hold the original object keep the shared write.
* fix(javascript): charge loader proofs through the wrapper-visit budget
#11319 replaced the wrapper-visit counter with a Budget, so the loader proof charge no longer compiled on current main. Spend it through the shared helper, which records the same limit.
* test(javascript): keep the alias bound off the quadratic write index
The 8,000 by 8,000 alias bound spent its time in the ambient write index, which walks every store to an object for each member write. That cost is the same on main and is not what the bound covers. Reach the same 64 million alias and write pairs with 32,000 aliases and 2,000 writes, which runs in about 2.5 seconds instead of 8.5 on the same host.
Drop the 30 second overrides on both alias bounds too. They halved the hosted CI deadline the source project gives them, and the testing guide asks tests to leave deadlines to the project.
* refactor(javascript): ask the effect log for loader proof effects
Main now guards reads of the effect indexes outside javascriptSemanticEffectLog.ts. The loader proof read them directly in 15 places. Route those reads through the log: the existing ambientPathMayHaveChanged and bindingHasMemberWrites queries, plus three timeless queries for a path or any member below it, an ambient binding write, and an escape at a site other than the call being proved. The answers are unchanged.
* fix(javascript): retain uncertainty for changed ambient values
---------
Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
575 lines
21 KiB
TypeScript
575 lines
21 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { cp, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
|
|
import { createPackage } from "@electron/asar";
|
|
|
|
import type { JsonValue } from "../../../src/domain/jsonValue.js";
|
|
import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js";
|
|
import type { ApplicationNode } from "../../../src/domain/javascript/javascriptApplicationGraphSchemas.js";
|
|
import { javaScriptExportShapeComparisonResultSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js";
|
|
import {
|
|
applicationFeatureTraceResultSchema,
|
|
featureSeedMatchMode,
|
|
} from "../../../src/domain/javascript/javascriptFeatureTraceSchemas.js";
|
|
import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascript/javascriptSemanticTraceSchemas.js";
|
|
import {
|
|
managedArtifactInspectionSchema,
|
|
managedMemberInspectionSchema,
|
|
} from "../../../src/domain/managed/managedArtifact.js";
|
|
import {
|
|
managedTokenScopesMatch,
|
|
readManagedInspectionEvidence,
|
|
} from "../../../src/domain/managed/managedInspectionEvidence.js";
|
|
import { compareUnicodeCodePoints } from "../../../src/domain/unicodeCodePointOrder.js";
|
|
import type {
|
|
FixtureClaimContext,
|
|
FixtureClaimExpectation,
|
|
} from "./KnownAnswerEvaluation.js";
|
|
|
|
/** Source-owned artifact paths used by the configured known-answer benchmarks. */
|
|
export interface AgentEvaluationFixtureTargets {
|
|
readonly javascript: string;
|
|
readonly javascriptSha256: string;
|
|
readonly javascriptShapeLeft: string;
|
|
readonly javascriptShapeRight: string;
|
|
readonly notesElectron: string;
|
|
readonly notesElectronSha256: string;
|
|
readonly managed?: string;
|
|
readonly managedSha256?: string;
|
|
}
|
|
|
|
const desktopFixtureFiles = {
|
|
"package.json": `${JSON.stringify({ name: "desktop-fixture", version: "1.0.0", main: "main.js" }, null, 2)}\n`,
|
|
"main.js":
|
|
'const { BrowserWindow, ipcMain } = require("electron");\nnew BrowserWindow({ webPreferences: { preload: require("node:path").join(__dirname, "preload.js"), contextIsolation: true, sandbox: true } });\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n',
|
|
"preload.js":
|
|
'const { contextBridge, ipcRenderer } = require("electron");\ncontextBridge.exposeInMainWorld("profileApi", { read: (id) => ipcRenderer.invoke("profile:read", id) });\n',
|
|
"renderer/app.js": 'globalThis.profileApi.read("fixture");\n',
|
|
};
|
|
|
|
/** Materialize the exact packaged and source-owned parser targets used by the runner. */
|
|
export const createAgentEvaluationFixtures = async (
|
|
root: string,
|
|
repositoryRoot: string,
|
|
): Promise<AgentEvaluationFixtureTargets> => {
|
|
const desktop = join(root, "desktop-app");
|
|
const javascript = join(root, "desktop-app.asar");
|
|
const javascriptShapeLeft = join(root, "parser-v1");
|
|
const javascriptShapeRight = join(root, "parser-v2");
|
|
const notesDirectory = join(root, "notes-electron");
|
|
const notesElectron = join(root, "notes-electron.asar");
|
|
await Promise.all([
|
|
mkdir(join(desktop, "renderer"), { recursive: true }),
|
|
mkdir(javascriptShapeLeft, { recursive: true }),
|
|
mkdir(javascriptShapeRight, { recursive: true }),
|
|
]);
|
|
await Promise.all([
|
|
...Object.entries(desktopFixtureFiles).map(([path, source]) =>
|
|
writeFile(join(desktop, path), source),
|
|
),
|
|
cp(
|
|
join(repositoryRoot, "tests/fixtures/replay/parser.mjs"),
|
|
join(javascriptShapeLeft, "parser.mjs"),
|
|
),
|
|
cp(
|
|
join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"),
|
|
join(javascriptShapeRight, "parser.mjs"),
|
|
),
|
|
cp(
|
|
join(repositoryRoot, "website/public/examples/notes-electron"),
|
|
notesDirectory,
|
|
{ recursive: true },
|
|
),
|
|
]);
|
|
await createPackage(desktop, javascript);
|
|
await createPackage(notesDirectory, notesElectron);
|
|
return {
|
|
javascript,
|
|
javascriptSha256: createHash("sha256")
|
|
.update(await readFile(javascript))
|
|
.digest("hex"),
|
|
javascriptShapeLeft,
|
|
javascriptShapeRight,
|
|
notesElectron,
|
|
notesElectronSha256: createHash("sha256")
|
|
.update(await readFile(notesElectron))
|
|
.digest("hex"),
|
|
};
|
|
};
|
|
|
|
type GraphObservation = ApplicationNode["observations"][number];
|
|
|
|
const singleObservation = (
|
|
value: JsonValue,
|
|
kind: ApplicationNode["kind"],
|
|
include: (observation: GraphObservation) => boolean = () => true,
|
|
): GraphObservation | undefined => {
|
|
const result = javascriptApplicationAnalysisResultSchema.safeParse(value);
|
|
if (!result.success) return undefined;
|
|
const candidates = result.data.graph.nodes
|
|
.filter((node) => node.kind === kind)
|
|
.flatMap((node) => node.observations)
|
|
.filter(include);
|
|
return candidates.length === 1 ? candidates[0] : undefined;
|
|
};
|
|
|
|
const observationSource = (
|
|
observation: GraphObservation,
|
|
): string | undefined => {
|
|
const location = observation.evidence.location;
|
|
return location.available && location.value.kind === "source-range"
|
|
? location.value.source
|
|
: undefined;
|
|
};
|
|
|
|
const rendererApi = (value: JsonValue): JsonValue | undefined => {
|
|
const observation = singleObservation(value, "context-bridge-api");
|
|
if (observation === undefined) return undefined;
|
|
const { api_key: key, members, world } = observation.properties;
|
|
const source = observationSource(observation);
|
|
if (
|
|
key === undefined ||
|
|
members === undefined ||
|
|
world === undefined ||
|
|
source === undefined
|
|
)
|
|
return undefined;
|
|
return { key, members, world, source };
|
|
};
|
|
|
|
const ipcOperation =
|
|
(side: string) =>
|
|
(value: JsonValue): JsonValue | undefined => {
|
|
const observation = singleObservation(
|
|
value,
|
|
"ipc-channel",
|
|
(candidate) => candidate.properties.side === side,
|
|
);
|
|
if (observation === undefined) return undefined;
|
|
const { channel, operation } = observation.properties;
|
|
const source = observationSource(observation);
|
|
if (
|
|
channel === undefined ||
|
|
operation === undefined ||
|
|
source === undefined
|
|
)
|
|
return undefined;
|
|
return { channel, operation, source };
|
|
};
|
|
|
|
const preload = (value: JsonValue): JsonValue | undefined => {
|
|
const observation = singleObservation(
|
|
value,
|
|
"electron-preload",
|
|
(candidate) =>
|
|
candidate.properties.mechanism === "BrowserWindow:webPreferences.preload",
|
|
);
|
|
const path = observation?.properties.resolved_path;
|
|
const resolutionStatus = observation?.properties.resolution_status;
|
|
if (path === undefined || resolutionStatus === undefined) return undefined;
|
|
return { path, resolution_status: resolutionStatus };
|
|
};
|
|
|
|
const headingChange = (value: JsonValue): JsonValue | undefined => {
|
|
const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
|
|
if (!result.success) return undefined;
|
|
const changes = result.data.changes.filter(
|
|
(change) => change.path === "/depth",
|
|
);
|
|
const change = changes.length === 1 ? changes[0] : undefined;
|
|
if (change === undefined) return undefined;
|
|
return {
|
|
status: change.status,
|
|
path: change.path,
|
|
discriminant: change.discriminant,
|
|
left: change.left,
|
|
right: change.right,
|
|
};
|
|
};
|
|
|
|
const comparisonCoverage = (value: JsonValue): JsonValue | undefined => {
|
|
const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
|
|
if (!result.success) return undefined;
|
|
return {
|
|
status: result.data.coverage.status,
|
|
paired_variants: result.data.coverage.paired_variants,
|
|
...result.data.summary,
|
|
};
|
|
};
|
|
|
|
const runtimeSemantics = (value: JsonValue): JsonValue | undefined => {
|
|
const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
|
|
return result.success &&
|
|
result.data.limitations.includes(
|
|
"Return shapes are inferred from inert syntax and do not prove runtime behavior.",
|
|
)
|
|
? "not_established"
|
|
: undefined;
|
|
};
|
|
|
|
const featureModules = (value: JsonValue): JsonValue | undefined => {
|
|
const result = applicationFeatureTraceResultSchema.safeParse(value);
|
|
if (!result.success || result.data.graph === null) return undefined;
|
|
if (
|
|
result.data.seed.kind !== "channel" ||
|
|
result.data.seed.value !== "notes:export" ||
|
|
featureSeedMatchMode(result.data.seed) !== "exact"
|
|
)
|
|
return undefined;
|
|
const paths = result.data.graph.nodes
|
|
.filter((node) => node.kind === "javascript-module")
|
|
.flatMap((node) => node.observations)
|
|
.filter(({ properties }) => properties.semantic_role === "source-module")
|
|
.map(({ properties }) => properties.path);
|
|
if (paths.length === 0 || !paths.every((path) => typeof path === "string"))
|
|
return undefined;
|
|
return {
|
|
paths: [...new Set(paths)].sort(compareUnicodeCodePoints),
|
|
coverage: result.data.coverage.status,
|
|
};
|
|
};
|
|
|
|
const semanticBoundary = (value: JsonValue): JsonValue | undefined => {
|
|
const result = javaScriptSemanticTraceResultSchema.safeParse(value);
|
|
if (!result.success) return undefined;
|
|
return {
|
|
status: result.data.status,
|
|
coverage: result.data.coverage.status,
|
|
relevant_unknowns: result.data.summary.relevant_unknowns,
|
|
};
|
|
};
|
|
|
|
const notesClaims = (
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] => {
|
|
const subject = {
|
|
path: targets.notesElectron,
|
|
sha256: targets.notesElectronSha256,
|
|
};
|
|
const derived = {
|
|
authority: "analyst-inference",
|
|
confidence: "inferred",
|
|
linkedSubjects: [
|
|
{ operation: "analyze_javascript_application", ...subject },
|
|
],
|
|
} satisfies Pick<
|
|
FixtureClaimExpectation["source"],
|
|
"authority" | "confidence" | "linkedSubjects"
|
|
>;
|
|
// Fixture source establishes bridge/module facts; the semantic claim checks
|
|
// the bounded static-query contract, not execution or CSV formatter behavior.
|
|
return [
|
|
...bridgeClaims(subject, "notes", ["exportCsv"], "notes:export"),
|
|
{
|
|
id: "feature_modules",
|
|
expectedValue: {
|
|
paths: ["csv.js", "main.js", "preload.js", "renderer.js"],
|
|
// The renderer's unresolved Promise member cannot hide an Electron
|
|
// call path, so it does not make this feature trace partial.
|
|
coverage: "complete-within-source",
|
|
},
|
|
source: {
|
|
...derived,
|
|
operation: "trace_application_feature",
|
|
parameters: {
|
|
direction: "both",
|
|
},
|
|
select: featureModules,
|
|
},
|
|
},
|
|
{
|
|
id: "semantic_boundary",
|
|
expectedValue: {
|
|
status: "ambiguous",
|
|
coverage: "partial",
|
|
relevant_unknowns: 1,
|
|
},
|
|
source: {
|
|
...derived,
|
|
operation: "trace_javascript_semantics",
|
|
parameters: {
|
|
query: {
|
|
seed: { kind: "property", name: "invoke" },
|
|
direction: "forward-influence",
|
|
include_ambiguous_dynamic_edges: false,
|
|
expected: null,
|
|
},
|
|
},
|
|
select: semanticBoundary,
|
|
},
|
|
},
|
|
];
|
|
};
|
|
|
|
const bridgeClaims = (
|
|
subject: { readonly path: string; readonly sha256: string },
|
|
key: string,
|
|
members: readonly string[],
|
|
channel: string,
|
|
): readonly FixtureClaimExpectation[] => {
|
|
const source = {
|
|
operation: "analyze_javascript_application",
|
|
authority: "shipped-artifact",
|
|
confidence: "derived",
|
|
subject,
|
|
} satisfies Omit<FixtureClaimExpectation["source"], "select">;
|
|
return [
|
|
{
|
|
id: "renderer_api",
|
|
expectedValue: {
|
|
key,
|
|
members: [...members],
|
|
world: "main",
|
|
source: "preload.js",
|
|
},
|
|
source: { ...source, select: rendererApi },
|
|
},
|
|
{
|
|
id: "renderer_ipc",
|
|
expectedValue: {
|
|
channel,
|
|
operation: "invoke",
|
|
source: "preload.js",
|
|
},
|
|
source: { ...source, select: ipcOperation("renderer") },
|
|
},
|
|
{
|
|
id: "main_ipc",
|
|
expectedValue: {
|
|
channel,
|
|
operation: "handle",
|
|
source: "main.js",
|
|
},
|
|
source: { ...source, select: ipcOperation("main") },
|
|
},
|
|
{
|
|
id: "preload",
|
|
expectedValue: { path: "preload.js", resolution_status: "resolved" },
|
|
source: { ...source, select: preload },
|
|
},
|
|
];
|
|
};
|
|
|
|
const asarClaims = (
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] =>
|
|
bridgeClaims(
|
|
{ path: targets.javascript, sha256: targets.javascriptSha256 },
|
|
"profileApi",
|
|
["read"],
|
|
"profile:read",
|
|
);
|
|
|
|
const parserClaims = (
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] => {
|
|
const source = {
|
|
operation: "compare_javascript_export_shapes",
|
|
authority: "analyst-inference",
|
|
confidence: "inferred",
|
|
parameters: {
|
|
left_module_path: "parser.mjs",
|
|
left_export_name: "default",
|
|
right_module_path: "parser.mjs",
|
|
right_export_name: "default",
|
|
},
|
|
linkedSubjects: [
|
|
{
|
|
operation: "analyze_javascript_application",
|
|
path: targets.javascriptShapeLeft,
|
|
},
|
|
{
|
|
operation: "analyze_javascript_application",
|
|
path: targets.javascriptShapeRight,
|
|
},
|
|
],
|
|
} satisfies Omit<FixtureClaimExpectation["source"], "select">;
|
|
return [
|
|
{
|
|
id: "heading_change",
|
|
expectedValue: {
|
|
status: "added",
|
|
path: "/depth",
|
|
discriminant: { path: "/type", value: "heading" },
|
|
left: { availability: "absent" },
|
|
right: { availability: "literal", value: 1 },
|
|
},
|
|
source: { ...source, select: headingChange },
|
|
},
|
|
{
|
|
id: "comparison_coverage",
|
|
expectedValue: {
|
|
status: "partial",
|
|
paired_variants: 3,
|
|
added: 1,
|
|
removed: 0,
|
|
changed: 0,
|
|
unknown: 3,
|
|
},
|
|
source: { ...source, select: comparisonCoverage },
|
|
},
|
|
{
|
|
id: "runtime_semantics",
|
|
expectedValue: "not_established",
|
|
source: { ...source, select: runtimeSemantics },
|
|
},
|
|
];
|
|
};
|
|
|
|
const managedMembers = ({ evidence }: FixtureClaimContext) => {
|
|
const parsed = readManagedInspectionEvidence(
|
|
evidence,
|
|
"inspect_managed_members",
|
|
managedMemberInspectionSchema,
|
|
{ operation: "agent_evaluation", path: ["evidence"] },
|
|
);
|
|
if (!parsed.ok) throw parsed.error;
|
|
const { result } = parsed.value;
|
|
if (
|
|
result.coverage.state !== "complete" ||
|
|
result.metadata.status !== "complete" ||
|
|
!managedTokenScopesMatch(
|
|
result.identity_scope.requires_mvid,
|
|
result.module?.mvid ?? null,
|
|
)
|
|
)
|
|
return undefined;
|
|
return result;
|
|
};
|
|
|
|
const managedClaims = (
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] => {
|
|
if (targets.managed === undefined || targets.managedSha256 === undefined)
|
|
throw new TypeError(
|
|
"Managed fixture grading requires the built DLL path and SHA-256",
|
|
);
|
|
const source = {
|
|
operation: "inspect_managed_members",
|
|
authority: "shipped-artifact",
|
|
confidence: "observed",
|
|
subject: { path: targets.managed, sha256: targets.managedSha256 },
|
|
} satisfies Omit<FixtureClaimExpectation["source"], "select">;
|
|
return [
|
|
{
|
|
id: "public_types",
|
|
expectedValue: ["AgentEval.Profile", "AgentEval.Program"],
|
|
source: {
|
|
...source,
|
|
select: (_value, context) => {
|
|
const members = managedMembers(context);
|
|
if (
|
|
members === undefined ||
|
|
members.selection !== null ||
|
|
Object.hasOwn(context.evidence.parameters, "type_names")
|
|
)
|
|
return undefined;
|
|
return members.types
|
|
.filter((type) => (type.flags & 7) === 1)
|
|
.map((type) => type.full_name)
|
|
.sort(compareUnicodeCodePoints);
|
|
},
|
|
},
|
|
},
|
|
{
|
|
id: "entry_point",
|
|
expectedValue: { declaring_type: "AgentEval.Program", name: "Main" },
|
|
source: {
|
|
...source,
|
|
select: (_value, context) => {
|
|
const members = managedMembers(context);
|
|
if (members === undefined) return undefined;
|
|
const artifacts = context.deliveredEvidence.filter(
|
|
(evidence) =>
|
|
evidence.operation === "inspect_managed_artifact" &&
|
|
evidence.authority === source.authority &&
|
|
evidence.confidence === source.confidence &&
|
|
evidence.subject?.local_path === source.subject.path &&
|
|
evidence.subject.identity.sha256 === source.subject.sha256,
|
|
);
|
|
if (artifacts.length !== 1) return undefined;
|
|
const parsedArtifact = readManagedInspectionEvidence(
|
|
artifacts[0],
|
|
"inspect_managed_artifact",
|
|
managedArtifactInspectionSchema,
|
|
{ operation: "agent_evaluation", path: ["deliveredEvidence"] },
|
|
);
|
|
if (!parsedArtifact.ok) throw parsedArtifact.error;
|
|
const { result: artifact } = parsedArtifact.value;
|
|
if (
|
|
artifact.coverage.state !== "complete" ||
|
|
artifact.metadata.status !== "complete" ||
|
|
!managedTokenScopesMatch(
|
|
artifact.module?.mvid ?? null,
|
|
members.module?.mvid ?? null,
|
|
) ||
|
|
artifact.pe.cli?.entry_point.kind !== "metadata-token"
|
|
)
|
|
return undefined;
|
|
const methods = members.methods.filter(
|
|
(method) => method.token === artifact.pe.cli?.entry_point.value,
|
|
);
|
|
const method = methods.length === 1 ? methods[0] : undefined;
|
|
return method === undefined
|
|
? undefined
|
|
: {
|
|
declaring_type: method.declaring_type,
|
|
name: method.name,
|
|
};
|
|
},
|
|
},
|
|
},
|
|
];
|
|
};
|
|
|
|
/** Return the closed known-answer rubric, or undefined for routing-only scenarios. */
|
|
export const agentFixtureClaims = (
|
|
scenarioId: string,
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] | undefined => {
|
|
if (scenarioId === "asar") return asarClaims(targets);
|
|
if (scenarioId === "managed") return managedClaims(targets);
|
|
if (scenarioId === "javascript-export-shape") return parserClaims(targets);
|
|
if (scenarioId === "notes-export") return notesClaims(targets);
|
|
return undefined;
|
|
};
|
|
|
|
const bridgeAnswerDescriptions = [
|
|
'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.',
|
|
'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',
|
|
'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.',
|
|
'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.',
|
|
];
|
|
|
|
const answerValueDescriptions: Readonly<Record<string, readonly string[]>> = {
|
|
managed: [
|
|
"public_types: string[], listing the exact public top-level type full names in Unicode code-point order from unfiltered member Evidence (omit type_names).",
|
|
'entry_point: {"declaring_type": string, "name": string}, resolving the CLI metadata entry-point token against the member inventory from the same artifact SHA-256 and module MVID. Cite member Evidence; both artifact and member Evidence must be delivered.',
|
|
],
|
|
asar: bridgeAnswerDescriptions,
|
|
"javascript-export-shape": [
|
|
'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.',
|
|
'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.',
|
|
'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.',
|
|
],
|
|
"notes-export": [
|
|
...bridgeAnswerDescriptions,
|
|
'feature_modules: {"paths": string[], "coverage": string}, listing the unique source-module observation paths in the feature trace in Unicode code-point order, and its coverage status.',
|
|
'semantic_boundary: {"status": string, "coverage": string, "relevant_unknowns": number}, copying the semantic trace status, coverage status, and summary.relevant_unknowns. Static traces do not establish runtime execution.',
|
|
],
|
|
};
|
|
|
|
/** Describe the strict answer contract without revealing any golden fixture values. */
|
|
export const agentFixtureAnswerInstructions = (scenarioId: string): string => {
|
|
const descriptions = answerValueDescriptions[scenarioId];
|
|
if (descriptions === undefined) return "";
|
|
return [
|
|
"For this benchmark, return only one JSON object, without Markdown fences or surrounding prose.",
|
|
'The exact shape is {"claims": [{"id": string, "value": JSON value, "evidence_id": string, "confidence": string, "authority": string}]}.',
|
|
"Include exactly one claim per ID below, without additional fields or claims. Infer values only from REA tool results; do not read target files directly.",
|
|
"Each evidence_id must identify the successful REA result that supports its claim. Copy confidence and authority from that result's Evidence envelope.",
|
|
...descriptions,
|
|
].join("\n");
|
|
};
|