From 84ee604c6d06b93cc5e139469e836e2283a4c09e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:57:30 +0800 Subject: [PATCH] perf(ci): optimize semantic graph construction, directory containment checks, gate checkouts, and boundary stragglers (#11851) * perf(javascript): avoid redundant semantic graph validation and node hashing * perf(resources): deduplicate directory containment checks and root lstat calls * perf(ci): trace script fixture modules, sparse-checkout gate jobs, and parallelize built doc checks * perf(tests): memoize live MCP doctor expectations and reuse CLI and verifier compilation * fix(ci): enforce containment after skipped entries, validate interned contexts, and supervise doc and verifier teardowns * fix(docs): decode buffered check output once to preserve multi-byte UTF-8 --- .github/workflows/ci.yml | 6 + .github/workflows/codeql.yml | 4 + package.json | 3 +- scripts/check-doc-facts.mjs | 194 ++++- scripts/ci/test-selection.mjs | 9 +- scripts/verify-documentation-routes.mjs | 36 +- .../JavaScriptSemanticGraphBuilder.ts | 11 +- .../JavaScriptSemanticGraphConstruction.ts | 94 +- .../JavaScriptSemanticGraphEvidence.ts | 108 ++- ...criptSemanticGraphFingerprintProjection.ts | 4 +- src/contracts/catalog/mcpDoctorCatalog.ts | 41 +- src/domain/canonicalDigest.ts | 32 +- .../javascriptApplicationEvidenceSchemas.ts | 72 +- .../javascript/javascriptSemanticGraph.ts | 822 +++++++++++++++--- .../javascriptSemanticGraphQuery.test.ts | 162 ++++ src/domain/zodCollectionSteps.ts | 5 + .../ReferenceSourceReaderValidate.ts | 2 +- src/resources/DirectoryArtifactReader.ts | 29 +- tests/boundary/cli/cliParityGoldens.test.ts | 1 - .../directoryArtifactReaderOwnership.test.ts | 83 +- .../filesystem/wideDirectoryTraversal.test.ts | 12 +- tests/boundary/mcp/mcpDoctor.test.ts | 12 +- .../ida/verifierObservations.test.ts | 63 +- .../providers/kuna/kunaVerifier.test.ts | 75 +- tests/evaluation/ciTestSelection.test.ts | 29 + 25 files changed, 1650 insertions(+), 259 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91380c8c9f..4306ae575c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1349,6 +1349,12 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + filter: blob:none + sparse-checkout: | + .nvmrc + scripts/ci + sparse-checkout-cone-mode: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 74dc8952fb..9e2d0c6ab6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -57,6 +57,10 @@ jobs: with: fetch-depth: ${{ github.event_name == 'pull_request' && 0 || 1 }} filter: blob:none + sparse-checkout: | + .nvmrc + scripts/ci + sparse-checkout-cone-mode: false persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: diff --git a/package.json b/package.json index 80c25dd7bd..fa7fc10c98 100644 --- a/package.json +++ b/package.json @@ -303,7 +303,8 @@ "docs:preview": "vitepress preview docs", "docs:generate": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts npm run docs:prepare", "docs:check": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run docs:check:built && npm run test:local -- tests/conformance/readmeConsistency.test.ts", - "docs:check:built": "node scripts/generate-package-metadata.mjs --check && node scripts/generate-skill-metadata.mjs --check && node scripts/generate-cursor-plugin-skill.mjs --check && node scripts/generate-product-catalog.mjs --check && node scripts/generate-completion-ledger.mjs --check && node scripts/check-doc-facts.mjs && node scripts/generate-error-schema.mjs --check && npm run licenses:check", + "docs:check:built": "node scripts/check-doc-facts.mjs --built", + "cursor-plugin:generate": "node scripts/generate-cursor-plugin-skill.mjs", "docs:error-schema": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run artifacts:error-schema", "prepack": "node scripts/prepack.mjs", "prepare": "node scripts/prepare.mjs", diff --git a/scripts/check-doc-facts.mjs b/scripts/check-doc-facts.mjs index 1c754c0140..53cbec4ffe 100644 --- a/scripts/check-doc-facts.mjs +++ b/scripts/check-doc-facts.mjs @@ -1,10 +1,196 @@ +import { spawn } from "node:child_process"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { syncCursorPluginSkill } from "./lib/cursor-plugin-bundle.mjs"; import { assertDocumentationFacts } from "./lib/docs-facts.mjs"; -import { createProductCatalog } from "./lib/product-catalog.mjs"; +import { ensureGeneratedFile } from "./lib/generated-file.mjs"; +import { + createProductCatalog, + serializeProductCatalog, +} from "./lib/product-catalog.mjs"; + +const MAX_OUTPUT_BYTES = 20 * 1024 * 1024; +const arguments_ = new Set(process.argv.slice(2)); +for (const argument of arguments_) + if (argument !== "--built") + throw new Error(`Unknown documentation check option: ${argument}`); -if (process.argv.length > 2) - throw new Error(`Unknown documentation check option: ${process.argv[2]}`); const root = join(dirname(fileURLToPath(import.meta.url)), ".."); -await assertDocumentationFacts(root, await createProductCatalog(root)); +if (!arguments_.has("--built")) { + await assertDocumentationFacts(root, await createProductCatalog(root)); +} else { + /** @type {Set<{ readonly stop: (signal: NodeJS.Signals) => void; readonly closed: Promise }>} */ + const activeChildren = new Set(); + /** @type {NodeJS.Signals | undefined} */ + let cancelledSignal; + + /** + * @param {import("node:child_process").ChildProcess} child + * @param {NodeJS.Signals} signal + */ + const signalChildGroup = (child, signal) => { + if (child.exitCode !== null || child.signalCode !== null) return; + try { + if (process.platform !== "win32" && child.pid !== undefined) + process.kill(-child.pid, signal); + else child.kill(signal); + } catch { + /* child already exited */ + } + }; + + /** @param {NodeJS.Signals} signal */ + const onTerminationSignal = (signal) => { + cancelledSignal ??= signal; + for (const active of activeChildren) active.stop(signal); + }; + const onSigint = () => onTerminationSignal("SIGINT"); + const onSigterm = () => onTerminationSignal("SIGTERM"); + process.on("SIGINT", onSigint); + process.on("SIGTERM", onSigterm); + + /** + * @param {string} script + * @returns {Promise<{ readonly stdout: string; readonly stderr: string }>} + */ + const runCheckScript = (script) => + new Promise((resolve, reject) => { + if (cancelledSignal !== undefined) { + reject( + new Error(`Documentation check cancelled by ${cancelledSignal}`), + ); + return; + } + const child = spawn(process.execPath, [join(root, script), "--check"], { + cwd: root, + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"], + }); + /** @type {Buffer[]} */ + const stdoutChunks = []; + /** @type {Buffer[]} */ + const stderrChunks = []; + let outputBytes = 0; + let overflowed = false; + /** @type {ReturnType | undefined} */ + let escalationTimer; + const stop = (/** @type {NodeJS.Signals} */ signal) => { + signalChildGroup(child, signal); + escalationTimer ??= setTimeout( + () => signalChildGroup(child, "SIGKILL"), + 2_000, + ); + escalationTimer.unref(); + }; + /** @type {() => void} */ + let markClosed = () => {}; + const closed = new Promise((resolveClosed) => { + markClosed = resolveClosed; + }); + const entry = { stop, closed }; + activeChildren.add(entry); + if (cancelledSignal !== undefined) stop(cancelledSignal); + + /** @param {Buffer[]} chunks @param {Buffer} chunk */ + const collect = (chunks, chunk) => { + outputBytes += chunk.byteLength; + if (outputBytes > MAX_OUTPUT_BYTES) { + if (!overflowed) { + overflowed = true; + stop("SIGTERM"); + } + return; + } + chunks.push(chunk); + }; + child.stdout.on("data", (chunk) => collect(stdoutChunks, chunk)); + child.stderr.on("data", (chunk) => collect(stderrChunks, chunk)); + child.once("error", (error) => { + if (escalationTimer !== undefined) clearTimeout(escalationTimer); + activeChildren.delete(entry); + markClosed(); + reject(error); + }); + child.once("close", (code, signal) => { + if (escalationTimer !== undefined) clearTimeout(escalationTimer); + activeChildren.delete(entry); + markClosed(); + const stdout = Buffer.concat(stdoutChunks).toString("utf8"); + const stderr = Buffer.concat(stderrChunks).toString("utf8"); + if (!overflowed && code === 0 && signal === null) { + resolve({ stdout, stderr }); + return; + } + const reason = overflowed + ? `${script} exceeded ${String(MAX_OUTPUT_BYTES)} bytes of diagnostic output` + : signal !== null + ? `${script} terminated by ${signal}` + : `${script} exited with code ${String(code)}`; + reject(Object.assign(new Error(reason), { stdout, stderr })); + }); + }); + + /** @type {PromiseSettledResult[]} */ + let settled = []; + try { + const catalogPromise = createProductCatalog(root); + settled = await Promise.allSettled([ + catalogPromise.then((catalog) => assertDocumentationFacts(root, catalog)), + catalogPromise.then(async (catalog) => + ensureGeneratedFile({ + path: join(root, "docs/public/product-catalog.json"), + source: await serializeProductCatalog(catalog), + check: true, + generateCommand: "npm run docs:generate", + }), + ), + syncCursorPluginSkill({ root, check: true }), + runCheckScript("scripts/generate-package-metadata.mjs"), + runCheckScript("scripts/generate-skill-metadata.mjs"), + runCheckScript("scripts/generate-completion-ledger.mjs"), + runCheckScript("scripts/generate-error-schema.mjs"), + runCheckScript("scripts/generate-third-party-inventory.mjs"), + ]); + } finally { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + for (const active of activeChildren) active.stop("SIGTERM"); + await Promise.allSettled([...activeChildren].map(({ closed }) => closed)); + } + + const failures = settled.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + for (const failure of failures) { + if (failure && typeof failure === "object") { + if ( + "stdout" in failure && + typeof failure.stdout === "string" && + failure.stdout + ) + process.stderr.write(failure.stdout); + if ( + "stderr" in failure && + typeof failure.stderr === "string" && + failure.stderr + ) + process.stderr.write(failure.stderr); + } + } + if (cancelledSignal !== undefined) { + process.exitCode = cancelledSignal === "SIGINT" ? 130 : 143; + throw new Error(`Documentation check cancelled by ${cancelledSignal}`); + } + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) { + throw new AggregateError( + failures, + failures + .map((cause) => + cause instanceof Error ? cause.message : String(cause), + ) + .join("\n"), + ); + } +} diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs index a0dc505a60..ffc85bd182 100644 --- a/scripts/ci/test-selection.mjs +++ b/scripts/ci/test-selection.mjs @@ -82,11 +82,12 @@ const explicitFixtureOwners = (path) => // Verifier scripts are imported by their tests like source modules. const isVerifier = (path) => /^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path); -// Shared script modules. A change is traced through the tests, verifiers and -// other shared modules that import it; any other consumer (build, test -// runners, configuration) shapes every run, so it selects the full suite. +// Shared script and fixture modules. A change is traced through the tests, +// verifiers and other shared modules that import it; any other consumer (build, +// test runners, configuration) shapes every run, so it selects the full suite. const isScriptLibrary = (path) => - !isCheckerInput(path) && /^scripts\/lib\/.+\.m?[jt]s$/u.test(path); + !isCheckerInput(path) && + /^scripts\/(?:lib|fixtures)\/.+\.m?[jt]s$/u.test(path); // Other scripts (build, test runners, CI) shape every run and are not traced. const isModule = (path) => !isCheckerInput(path) && diff --git a/scripts/verify-documentation-routes.mjs b/scripts/verify-documentation-routes.mjs index e9db4256b0..a4e41f18f8 100644 --- a/scripts/verify-documentation-routes.mjs +++ b/scripts/verify-documentation-routes.mjs @@ -53,12 +53,29 @@ while (directories.length > 0) { htmlPages.push(path); } } -for (const page of htmlPages.sort()) { +htmlPages.sort(); +const READ_CONCURRENCY = 32; +/** @type {Map} */ +const htmlByPath = new Map(); +let nextPageIndex = 0; +await Promise.all( + Array.from( + { length: Math.min(READ_CONCURRENCY, htmlPages.length) }, + async () => { + while (nextPageIndex < htmlPages.length) { + const page = htmlPages[nextPageIndex++]; + htmlByPath.set(page, await readFile(page, "utf8")); + } + }, + ), +); +for (const page of htmlPages) { const route = page .slice(output.length + 1) .split(sep) .join("/"); - const html = await readFile(page, "utf8"); + const html = htmlByPath.get(page); + assert.ok(html !== undefined, route + ": missing cached HTML page"); // The configured home layout and generated 404 render no edit action. // Their outgoing links still participate in the same all-page scan. if (route !== "404.html" && route !== "index.html") { @@ -90,15 +107,18 @@ for (const page of htmlPages.sort()) { physical.startsWith(output + sep), destination + ": published path escaped site output", ); - assert.ok( - (await stat(physical)).isFile(), - route + ": broken rendered target " + destination, - ); + const cachedTargetHtml = htmlByPath.get(physical); + if (cachedTargetHtml === undefined) { + assert.ok( + (await stat(physical)).isFile(), + route + ": broken rendered target " + destination, + ); + } if (url.hash && target.endsWith(".html")) { const fragment = decodeURIComponent(url.hash.slice(1)); - const page = await readFile(physical, "utf8"); + const targetHtml = cachedTargetHtml ?? (await readFile(physical, "utf8")); assert.ok( - page.includes('id="' + fragment + '"'), + targetHtml.includes('id="' + fragment + '"'), route + ": missing rendered anchor " + destination, ); } diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 4b73f1828c..3e64f9caf9 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -84,6 +84,7 @@ import { addSemanticGraphRelation as addRelation, createSemanticGraphProjectionState as emptyState, bindSemanticGraphApplicationNodes, + semanticNodeIdentityLookupKey, type SemanticGraphProjectionState as BuilderState, } from "./JavaScriptSemanticGraphConstruction.js"; import type { @@ -332,7 +333,7 @@ export const createJavaScriptSemanticGraphProjection = ( root_artifact_sha256: rootArtifactSha256, application_graph_id: applicationGraph.graph_id, root_node_ids: [...state.roots], - evidence_contexts: state.evidenceContexts.contexts, + evidence_contexts: state.evidenceContexts.ownedContexts, nodes: [...state.nodes.values()], relations, // The input is transferred to the factory; `fingerprints` is cleared below. @@ -389,6 +390,7 @@ export const createJavaScriptSemanticGraphProjection = ( ], }); state.nodes.clear(); + state.nodesByLookupKey.clear(); state.relations.clear(); state.unknowns.clear(); state.unknownOmissions.clear(); @@ -437,7 +439,10 @@ export const createJavaScriptSemanticGraphProjection = ( recordSemanticUnknownOmission(state, omission); for (const context of projection.evidenceContexts) state.evidenceContexts.adopt(context); - for (const node of newNodes) state.nodes.set(node.node_id, node); + for (const node of newNodes) { + state.nodes.set(node.node_id, node); + state.nodesByLookupKey.set(semanticNodeIdentityLookupKey(node), node); + } for (const relation of projection.relations) state.relations.set(relation.relation_id, relation); for (const unknown of newUnknowns) @@ -582,7 +587,7 @@ export function* projectJavaScriptSemanticFileSteps( const fingerprints = yield* projectFileSteps(file, ir, state, paths); return { roots: [...state.roots], - evidenceContexts: state.evidenceContexts.contexts, + evidenceContexts: state.evidenceContexts.ownedContexts, nodes: [...state.nodes.values()], relations: [...state.relations.values()], unknowns: [...state.unknowns.values()], diff --git a/src/application/javascript/JavaScriptSemanticGraphConstruction.ts b/src/application/javascript/JavaScriptSemanticGraphConstruction.ts index 0377e281a6..f0979c3858 100644 --- a/src/application/javascript/JavaScriptSemanticGraphConstruction.ts +++ b/src/application/javascript/JavaScriptSemanticGraphConstruction.ts @@ -4,11 +4,11 @@ import type { JavaScriptApplicationGraph } from "../../domain/javascript/javascr import type { JsonValue } from "../../domain/jsonValue.js"; import { javascriptDisplayText } from "../../domain/javascript/javascriptAstValues.js"; import { - createJavaScriptSemanticGraphNode, - createJavaScriptSemanticGraphUnknown, - createJavaScriptSemanticGraphRelation, + createOwnedJavaScriptSemanticGraphNode, + createOwnedJavaScriptSemanticGraphUnknown, + createOwnedJavaScriptSemanticGraphRelation, JavaScriptSemanticEvidenceContextRegistry, - javaScriptSemanticNodeId, + withJavaScriptSemanticNodeApplicationIds, } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { JavaScriptSemanticGraphNode, @@ -29,6 +29,7 @@ import { /** Mutable local projection state hidden from graph callers. */ export interface SemanticGraphProjectionState { readonly nodes: Map; + readonly nodesByLookupKey: Map; readonly relations: Map; readonly unknowns: Map; /** Absolute retained unknown count allowed during this file; null outside admission. */ @@ -82,6 +83,7 @@ export interface SemanticRelationConstructionInput { export const createSemanticGraphProjectionState = (): SemanticGraphProjectionState => ({ nodes: new Map(), + nodesByLookupKey: new Map(), relations: new Map(), unknowns: new Map(), unknownCapacity: null, @@ -99,18 +101,26 @@ export const bindSemanticGraphApplicationNodes = ( applicationGraph.nodes, ); for (const [identifier, node] of state.nodes) { - state.nodes.set(identifier, { - ...node, - application_node_ids: [ - ...(applicationNodeIdsByLocation.get( - applicationLocationKey( - node.identity.artifact_sha256, - node.identity.module_path, - node.identity.source_range, - ), - ) ?? []), - ], - }); + const matched = applicationNodeIdsByLocation.get( + applicationLocationKey( + node.identity.artifact_sha256, + node.identity.module_path, + node.identity.source_range, + ), + ); + if (matched === undefined && node.application_node_ids.length === 0) + continue; + const updated = withJavaScriptSemanticNodeApplicationIds( + node, + matched ?? [], + ); + if (updated !== node) { + state.nodes.set(identifier, updated); + state.nodesByLookupKey.set( + semanticNodeIdentityLookupKey(updated), + updated, + ); + } } }; @@ -120,7 +130,7 @@ export const constructSemanticGraphNode = ( input: SemanticNodeConstructionInput, state: SemanticGraphProjectionState, ): JavaScriptSemanticGraphNode => - createJavaScriptSemanticGraphNode( + createOwnedJavaScriptSemanticGraphNode( { kind: input.kind, identity: { @@ -139,22 +149,40 @@ export const constructSemanticGraphNode = ( state.evidenceContexts, ); +const semanticNodeLookupKey = ( + kind: JavaScriptSemanticGraphNode["kind"], + artifactSha256: string, + modulePath: string, + sourceRange: JavaScriptSourceRange | null, + roleKey: string, +): string => + `${kind}:${artifactSha256}:${String(modulePath.length)}:${modulePath}:${sourceRange === null ? "-" : javascriptSourceRangeKey(sourceRange)}:${roleKey}`; + +export const semanticNodeIdentityLookupKey = ( + node: Pick, +): string => + semanticNodeLookupKey( + node.kind, + node.identity.artifact_sha256, + node.identity.module_path, + node.identity.source_range, + node.identity.role_key, + ); + /** Reuse retained identities and reject exhausted budgets before allocation. */ export const retainSemanticGraphNode = ( state: SemanticGraphProjectionState, file: JavaScriptArtifactFile, input: SemanticNodeConstructionInput, ): JavaScriptSemanticGraphNode | null => { - const identifier = javaScriptSemanticNodeId({ - kind: input.kind, - identity: { - artifact_sha256: file.sha256, - module_path: file.path, - source_range: input.location, - role_key: input.roleKey, - }, - }); - const existing = state.nodes.get(identifier); + const lookupKey = semanticNodeLookupKey( + input.kind, + file.sha256, + file.path, + input.location, + input.roleKey, + ); + const existing = state.nodesByLookupKey.get(lookupKey); if (existing !== undefined) return existing; if (typeof state.fileNodeBudget === "number" && state.fileNodeBudget <= 0) { state.fileNodesDropped = true; @@ -163,6 +191,7 @@ export const retainSemanticGraphNode = ( return addSemanticGraphNode( state, constructSemanticGraphNode(file, input, state), + lookupKey, ); }; @@ -170,6 +199,7 @@ export const retainSemanticGraphNode = ( export const addSemanticGraphNode = ( state: SemanticGraphProjectionState, node: JavaScriptSemanticGraphNode, + knownLookupKey?: string, ): JavaScriptSemanticGraphNode | null => { const existing = state.nodes.get(node.node_id); if (existing !== undefined) return existing; @@ -182,6 +212,10 @@ export const addSemanticGraphNode = ( state.fileNodeBudget = budget - 1; } state.nodes.set(node.node_id, node); + state.nodesByLookupKey.set( + knownLookupKey ?? semanticNodeIdentityLookupKey(node), + node, + ); return node; }; @@ -198,7 +232,7 @@ export const addSemanticGraphRelation = ( input.source.node_id === input.target.node_id ) return; - const value = createJavaScriptSemanticGraphRelation( + const value = createOwnedJavaScriptSemanticGraphRelation( { source_node_id: input.source.node_id, target_node_id: input.target.node_id, @@ -252,7 +286,7 @@ export const addSemanticGraphUnknown = ( }); return; } - const unknown = createJavaScriptSemanticGraphUnknown( + const unknown = createOwnedJavaScriptSemanticGraphUnknown( input, state.evidenceContexts, ); @@ -267,7 +301,7 @@ export const addSemanticFallbackRoot = ( ): void => { const node = addSemanticGraphNode( state, - createJavaScriptSemanticGraphNode( + createOwnedJavaScriptSemanticGraphNode( { kind: "module", identity: { diff --git a/src/application/javascript/JavaScriptSemanticGraphEvidence.ts b/src/application/javascript/JavaScriptSemanticGraphEvidence.ts index ccc5203375..c4bc98b80a 100644 --- a/src/application/javascript/JavaScriptSemanticGraphEvidence.ts +++ b/src/application/javascript/JavaScriptSemanticGraphEvidence.ts @@ -5,6 +5,66 @@ import type { JsonValue } from "../../domain/jsonValue.js"; const INFERENCE_LIMITATION = "This relationship is a conservative static candidate; it does not prove runtime execution or causal flow."; +const CANDIDATE_BUILTIN_LIMITATION = + "The built-in identity or its mutation state is unproven; this operation is conditional."; +const UNKNOWN_SYNTAX_LIMITATION = + "Dynamic syntax prevents an exact static relationship."; + +const SEMANTIC_EXTRACTOR: ApplicationGraphEvidence["extractor"] = Object.freeze( + { + name: "rea-javascript-semantic-analysis", + version: "1", + operation: "analyze_javascript_application", + executable_sha256: null, + }, +); + +const EMPTY_LIMITS = Object.freeze( + [], +) as unknown as ApplicationGraphEvidence["coverage"]["limits"]; + +const COMPLETE_COVERAGE: ApplicationGraphEvidence["coverage"] = Object.freeze({ + status: "complete", + truncated: false, + omitted_count: 0, + limits: EMPTY_LIMITS, +}); + +const EMPTY_STRINGS: readonly string[] = Object.freeze([]); +const INFERENCE_LIMITATIONS: readonly string[] = Object.freeze([ + INFERENCE_LIMITATION, +]); +const CANDIDATE_BUILTIN_LIMITATIONS: readonly string[] = Object.freeze([ + INFERENCE_LIMITATION, + CANDIDATE_BUILTIN_LIMITATION, +]); +const UNKNOWN_LIMITATIONS: readonly string[] = Object.freeze([ + UNKNOWN_SYNTAX_LIMITATION, +]); + +type AvailableArtifactReference = Extract< + ApplicationGraphEvidence["artifact"], + { readonly available: true } +>; + +const cachedFileArtifacts = new WeakMap< + JavaScriptArtifactFile, + AvailableArtifactReference +>(); + +const observedFileArtifact = ( + file: JavaScriptArtifactFile, +): AvailableArtifactReference => { + const cached = cachedFileArtifacts.get(file); + if (cached !== undefined && cached.sha256 === file.sha256) return cached; + const created: AvailableArtifactReference = Object.freeze({ + available: true, + artifact_id: `art_${file.sha256}`, + sha256: file.sha256, + }); + cachedFileArtifacts.set(file, created); + return created; +}; /** Exact artifact-backed syntax evidence for one semantic graph node. */ export const observedSemanticEvidence = ( @@ -14,11 +74,7 @@ export const observedSemanticEvidence = ( authority: "ast-static-analysis", state: "observed", confidence: "exact", - artifact: { - available: true, - artifact_id: `art_${file.sha256}`, - sha256: file.sha256, - }, + artifact: observedFileArtifact(file), location: location === null ? { available: true, value: { kind: "artifact-path", path: file.path } } @@ -26,20 +82,10 @@ export const observedSemanticEvidence = ( available: true, value: { kind: "source-range", source: file.path, ...location }, }, - extractor: { - name: "rea-javascript-semantic-analysis", - version: "1", - operation: "analyze_javascript_application", - executable_sha256: null, - }, - coverage: { - status: "complete", - truncated: false, - omitted_count: 0, - limits: [], - }, - limitations: [], - evidence_ids: [], + extractor: SEMANTIC_EXTRACTOR, + coverage: COMPLETE_COVERAGE, + limitations: EMPTY_STRINGS as string[], + evidence_ids: EMPTY_STRINGS as string[], }); /** Conservative static relationship evidence derived from one syntax node. */ @@ -50,7 +96,7 @@ export const inferredSemanticEvidence = ( authority: "static-relationship-inference", state: "inferred", confidence: "high", - limitations: [INFERENCE_LIMITATION], + limitations: INFERENCE_LIMITATIONS as string[], }); /** Static relationship evidence anchored at the syntax that proves the link. */ @@ -62,24 +108,20 @@ export const inferredSemanticEvidenceAt = ( authority: "static-relationship-inference", state: "inferred", confidence: "high", - limitations: [INFERENCE_LIMITATION], + limitations: INFERENCE_LIMITATIONS as string[], }); /** A built-in operation whose identity is conditional on loader or mutation facts. */ export const candidateBuiltinSemanticEvidence = ( file: JavaScriptArtifactFile, location: JavaScriptSourceRange, -): ApplicationGraphEvidence => { - const evidence = inferredSemanticEvidenceAt(file, location); - return { - ...evidence, - confidence: "low", - limitations: [ - ...evidence.limitations, - "The built-in identity or its mutation state is unproven; this operation is conditional.", - ], - }; -}; +): ApplicationGraphEvidence => ({ + ...observedSemanticEvidence(file, location), + authority: "static-relationship-inference", + state: "inferred", + confidence: "low", + limitations: CANDIDATE_BUILTIN_LIMITATIONS as string[], +}); type SemanticBuiltinOperation = { readonly builtinAuthority?: "candidate" | undefined; @@ -160,5 +202,5 @@ export const unknownSemanticEvidence = ( authority: "unknown", state: "unknown", confidence: "unknown", - limitations: ["Dynamic syntax prevents an exact static relationship."], + limitations: UNKNOWN_LIMITATIONS as string[], }); diff --git a/src/application/javascript/JavaScriptSemanticGraphFingerprintProjection.ts b/src/application/javascript/JavaScriptSemanticGraphFingerprintProjection.ts index 10b7ab495c..2c4f15279e 100644 --- a/src/application/javascript/JavaScriptSemanticGraphFingerprintProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphFingerprintProjection.ts @@ -1,5 +1,5 @@ import { - createJavaScriptSemanticFingerprint, + createOwnedJavaScriptSemanticFingerprint, type JavaScriptSemanticEvidenceContextRegistry, } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { @@ -21,7 +21,7 @@ export const projectSemanticFunctionFingerprints = ( const callable = callableNodes.get(fingerprint.callableId); if (callable === undefined) return []; return [ - createJavaScriptSemanticFingerprint( + createOwnedJavaScriptSemanticFingerprint( { function_node_id: callable.node_id, algorithm: "rea.javascript-semantic-function", diff --git a/src/contracts/catalog/mcpDoctorCatalog.ts b/src/contracts/catalog/mcpDoctorCatalog.ts index 2b0f4d2a59..5c105b5f94 100644 --- a/src/contracts/catalog/mcpDoctorCatalog.ts +++ b/src/contracts/catalog/mcpDoctorCatalog.ts @@ -18,12 +18,25 @@ export interface McpDoctorCatalogExpectation { readonly binarySessionOutputSchema: Record; } +const liveExpectationCache = new Map(); +let cachedBinarySessionOutputSchema: Record | undefined; + +const binarySessionOutputJsonSchema = (): Record => + (cachedBinarySessionOutputSchema ??= Object.freeze( + advertisedOutputJsonSchema(toolContract("binary_session")), + )); + /** Build expectations from live contracts when family selection or source runs require it. */ export const buildLiveMcpDoctorExpectation = ( selection: ReadonlySet | undefined, outputSchemaProfile: McpOutputSchemaProfile, progressive: boolean, ): McpDoctorCatalogExpectation => { + const key = `${ + selection === undefined ? "*" : [...selection].sort().join(",") + }|${outputSchemaProfile}|${progressive ? "1" : "0"}`; + const cached = liveExpectationCache.get(key); + if (cached !== undefined) return cached; const contracts = contractsForToolFamilies(selection); const promptContracts = promptsForToolFamilies(selection); const catalog = createCatalogIdentity({ @@ -31,19 +44,21 @@ export const buildLiveMcpDoctorExpectation = ( outputSchemaProfile, promptContracts, }); - return { + const expectation: McpDoctorCatalogExpectation = Object.freeze({ catalog: progressive ? progressiveCatalogIdentity(catalog) : catalog, - expectedToolNames: progressive - ? [ - ...contracts - .map(({ name }) => name) - .filter((name) => HOT_TOOL_NAMES.has(name)), - DISCOVER_TOOLS.name, - ] - : contracts.map(({ name }) => name), - expectedPromptNames: promptContracts.map(({ name }) => name), - binarySessionOutputSchema: advertisedOutputJsonSchema( - toolContract("binary_session"), + expectedToolNames: Object.freeze( + progressive + ? [ + ...contracts + .map(({ name }) => name) + .filter((name) => HOT_TOOL_NAMES.has(name)), + DISCOVER_TOOLS.name, + ] + : contracts.map(({ name }) => name), ), - }; + expectedPromptNames: Object.freeze(promptContracts.map(({ name }) => name)), + binarySessionOutputSchema: binarySessionOutputJsonSchema(), + }); + liveExpectationCache.set(key, expectation); + return expectation; }; diff --git a/src/domain/canonicalDigest.ts b/src/domain/canonicalDigest.ts index f14ed309a1..ff2446aa05 100644 --- a/src/domain/canonicalDigest.ts +++ b/src/domain/canonicalDigest.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { sha256Hex } from "../kernel/sha256Hex.js"; /** * Streaming SHA-256 over canonical JSON. Hashing owner (avoids assembling @@ -10,6 +11,14 @@ export const digestCanonicalValue = ( value: unknown, context = "Comparison", ): string => { + if (typeof value === "number" && !Number.isFinite(value)) + throw new CanonicalDigestError(Number.isNaN(value) ? "nan" : "infinity"); + if (value === null || typeof value !== "object") { + const encoded = JSON.stringify(value); + if (encoded === undefined) + throw new TypeError(`${context} could not canonicalize data`); + return sha256Hex(encoded); + } const hash = createHash("sha256"); // Batch small emitted parts to cut hash.update crossings. A large part is // hashed as is: appending it to the buffer would hold a second copy of it @@ -86,14 +95,23 @@ const emitCanonical = ( } else { emit("{"); let first = true; - // Match canonicalize/RFC 8785, including supplementary-plane keys. - for (const key of Object.keys(value).sort()) { - if ( - Reflect.get(value, key) === undefined || - typeof Reflect.get(value, key) === "symbol" - ) + const record = value as Readonly>; + const keys = Object.keys(record); + if (keys.length === 2) { + if (keys[0]! > keys[1]!) { + const firstKey = keys[0]!; + keys[0] = keys[1]!; + keys[1] = firstKey; + } + } else if (keys.length > 2) { + // Match canonicalize/RFC 8785, including supplementary-plane keys. + keys.sort(); + } + for (let index = 0; index < keys.length; index += 1) { + const key = keys[index]!; + if (record[key] === undefined || typeof record[key] === "symbol") continue; - const item: unknown = Reflect.get(value, key); + const item: unknown = record[key]; if (!first) emit(","); first = false; emit(JSON.stringify(key)); diff --git a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts index fd53b23c85..b27176bfbb 100644 --- a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts +++ b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts @@ -270,10 +270,20 @@ export const applicationGraphEvidenceContextSchema = applicationGraphEvidenceBaseSchema.omit({ location: true }); type EvidenceValue = z.infer; +type EvidenceContextValue = z.infer< + typeof applicationGraphEvidenceContextSchema +>; +type EvidenceIssueSink = { + addIssue(issue: { + readonly code: "custom"; + readonly path?: PropertyKey[]; + readonly message: string; + }): void; +}; const checkAuthorityState = ( - evidence: EvidenceValue, - context: z.RefinementCtx, + evidence: EvidenceContextValue, + context: EvidenceIssueSink, ): void => { if ( evidence.authority === "static-relationship-inference" && @@ -314,8 +324,8 @@ const checkAuthorityState = ( }; const checkStateConfidence = ( - evidence: EvidenceValue, - context: z.RefinementCtx, + evidence: EvidenceContextValue, + context: EvidenceIssueSink, ): void => { if ( ["unknown", "unavailable"].includes(evidence.state) && @@ -349,16 +359,10 @@ const checkStateConfidence = ( }); }; -const checkTraceability = ( - evidence: EvidenceValue, - context: z.RefinementCtx, +const checkContextTraceability = ( + evidence: EvidenceContextValue, + context: EvidenceIssueSink, ): void => { - if (evidence.state === "observed" && !evidence.location.available) - context.addIssue({ - code: "custom", - path: ["location"], - message: "Observed facts require an actionable location", - }); if (evidence.state === "inferred" && evidence.limitations.length === 0) context.addIssue({ code: "custom", @@ -376,9 +380,30 @@ const checkTraceability = ( }); }; -const checkArtifactBacking = ( +/** Whether a validated evidence context permits the fact's source location. */ +export const isCompatibleApplicationGraphEvidenceLocation = ( + evidence: Pick, + location: z.infer, +): boolean => evidence.state !== "observed" || location.available; + +const checkTraceability = ( evidence: EvidenceValue, - context: z.RefinementCtx, + context: EvidenceIssueSink, +): void => { + if ( + !isCompatibleApplicationGraphEvidenceLocation(evidence, evidence.location) + ) + context.addIssue({ + code: "custom", + path: ["location"], + message: "Observed facts require an actionable location", + }); + checkContextTraceability(evidence, context); +}; + +const checkArtifactBacking = ( + evidence: EvidenceContextValue, + context: EvidenceIssueSink, ): void => { const artifactBacked = [ "artifact-bytes", @@ -399,6 +424,23 @@ const checkArtifactBacking = ( }); }; +/** Whether shared non-location provenance satisfies all context-level invariants. */ +export const isValidApplicationGraphEvidenceContext = ( + evidence: EvidenceContextValue, +): boolean => { + let valid = true; + const sink: EvidenceIssueSink = { + addIssue: () => { + valid = false; + }, + }; + checkAuthorityState(evidence, sink); + checkStateConfidence(evidence, sink); + checkContextTraceability(evidence, sink); + checkArtifactBacking(evidence, sink); + return valid; +}; + /** Provenance, authority, uncertainty, and bounds for one graph fact. */ export const applicationGraphEvidenceSchema = applicationGraphEvidenceBaseSchema.superRefine((evidence, context) => { diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index c9eafc671f..dbb509b969 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -5,26 +5,39 @@ import { digestCanonicalValue } from "../canonicalDigest.js"; import { uniqueSorted } from "../canonicalOrdering.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js"; -import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js"; +import { + freezeOwnedJsonSnapshotSteps, + freezeOwnedJsonTreeSnapshot, +} from "../immutableJson.js"; import { arrayContainerSchema, parseElementSteps, } from "../zodCollectionSteps.js"; +import { isRecord } from "../../kernel/isRecord.js"; import type { ApplicationGraphEvidence, ApplicationGraphEvidenceContext, } from "./javascriptApplicationEvidenceSchemas.js"; -import { applicationGraphEvidenceSchema } from "./javascriptApplicationEvidenceSchemas.js"; +import { + applicationGraphEvidenceLocationSchema, + applicationGraphEvidenceSchema, + isCompatibleApplicationGraphEvidenceLocation, + isValidApplicationGraphEvidenceContext, +} from "./javascriptApplicationEvidenceSchemas.js"; import { JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, JAVASCRIPT_SEMANTIC_RELATION_FAMILY, javaScriptSemanticFingerprintInputSchema, + javaScriptSemanticFingerprintSchema, javaScriptSemanticEvidenceContextSchema, javaScriptSemanticGraphInputSchema, javaScriptSemanticGraphRecordSchema, javaScriptSemanticNodeInputSchema, + javaScriptSemanticNodeSchema, javaScriptSemanticRelationInputSchema, + javaScriptSemanticRelationSchema, javaScriptSemanticUnknownInputSchema, + javaScriptSemanticUnknownSchema, type JavaScriptSemanticGraphInput, type JavaScriptSemanticGraphNode, type JavaScriptSemanticGraphRelation, @@ -33,55 +46,190 @@ import { type JavaScriptSemanticEvidenceContext, type JavaScriptSemanticEvidenceReference, } from "./javascriptSemanticGraphSchemas.js"; +import { prefixedDigestSchema } from "../digests.js"; const normalizeEvidence = ( evidence: ApplicationGraphEvidence, ): ApplicationGraphEvidence => ({ ...evidence, - coverage: { - ...evidence.coverage, - limits: [...evidence.coverage.limits].sort((left, right) => - compareUnicodeCodePoints( - canonicalJson(left, "JavaScript semantic graph"), - canonicalJson(right, "JavaScript semantic graph"), - ), - ), - }, - limitations: uniqueSorted(evidence.limitations), - evidence_ids: uniqueSorted(evidence.evidence_ids), + coverage: + evidence.coverage.limits.length <= 1 + ? { + ...evidence.coverage, + limits: [...evidence.coverage.limits], + } + : { + ...evidence.coverage, + limits: [...evidence.coverage.limits].sort((left, right) => + compareUnicodeCodePoints( + canonicalJson(left, "JavaScript semantic graph"), + canonicalJson(right, "JavaScript semantic graph"), + ), + ), + }, + limitations: + evidence.limitations.length <= 1 + ? [...evidence.limitations] + : uniqueSorted(evidence.limitations), + evidence_ids: + evidence.evidence_ids.length <= 1 + ? [...evidence.evidence_ids] + : uniqueSorted(evidence.evidence_ids), }); const evidenceContextId = (context: ApplicationGraphEvidenceContext): string => `jsrg_evidence_${digestCanonicalValue(context, "JavaScript semantic evidence context")}`; +interface FrozenInternedEvidenceEntry { + readonly authority: ApplicationGraphEvidence["authority"]; + readonly state: ApplicationGraphEvidence["state"]; + readonly confidence: ApplicationGraphEvidence["confidence"]; + readonly artifact: ApplicationGraphEvidence["artifact"]; + readonly extractor: ApplicationGraphEvidence["extractor"]; + readonly coverage: ApplicationGraphEvidence["coverage"]; + readonly limitations: readonly string[]; + readonly evidenceIds: readonly string[]; + readonly contextId: string; +} + +const hasReusableFrozenContext = ( + evidence: ApplicationGraphEvidence, +): boolean => + Object.isFrozen(evidence.artifact) && + Object.isFrozen(evidence.extractor) && + Object.isFrozen(evidence.coverage) && + Object.isFrozen(evidence.coverage.limits) && + evidence.coverage.limits.length === 0 && + Object.isFrozen(evidence.limitations) && + evidence.limitations.every((item) => typeof item === "string") && + Object.isFrozen(evidence.evidence_ids) && + evidence.evidence_ids.length === 0; + +const EVIDENCE_INPUT_KEYS = [ + "authority", + "state", + "confidence", + "artifact", + "location", + "extractor", + "coverage", + "limitations", + "evidence_ids", +] as const; + +const hasExactEvidenceKeys = ( + value: unknown, +): value is Record<(typeof EVIDENCE_INPUT_KEYS)[number], unknown> => + isRecord(value) && + Object.keys(value).length === EVIDENCE_INPUT_KEYS.length && + EVIDENCE_INPUT_KEYS.every((key) => Object.hasOwn(value, key)); + +const tryInternValidatedInputSymbol = Symbol("tryInternValidatedInput"); +const internValidatedEvidenceSymbol = Symbol("internValidatedEvidence"); + /** Graph-scoped owner for canonical provenance shared by semantic facts. */ export class JavaScriptSemanticEvidenceContextRegistry { readonly #contexts = new Map(); // Facts share a few contexts, so key them by canonical text and hash each // distinct context once instead of on every fact. readonly #idsByCanonicalContext = new Map(); + readonly #frozenEntries: FrozenInternedEvidenceEntry[] = []; - /** Intern normalized non-location provenance at fact construction. */ + /** Fast-path interning when shared context objects match an already-validated frozen entry. */ + [tryInternValidatedInputSymbol]( + rawEvidence: unknown, + ): JavaScriptSemanticEvidenceReference | undefined { + if (!hasExactEvidenceKeys(rawEvidence)) return undefined; + const match = this.#frozenEntries.find( + (entry) => + entry.authority === rawEvidence.authority && + entry.state === rawEvidence.state && + entry.confidence === rawEvidence.confidence && + entry.artifact === rawEvidence.artifact && + entry.extractor === rawEvidence.extractor && + entry.coverage === rawEvidence.coverage && + entry.limitations === rawEvidence.limitations && + entry.evidenceIds === rawEvidence.evidence_ids, + ); + if (match === undefined) return undefined; + const parsedLocation = applicationGraphEvidenceLocationSchema.safeParse( + rawEvidence.location, + ); + if ( + !parsedLocation.success || + !isCompatibleApplicationGraphEvidenceLocation(match, parsedLocation.data) + ) + return undefined; + return { context_id: match.contextId, location: parsedLocation.data }; + } + + /** Validate and intern non-location provenance at fact construction. */ intern( evidence: ApplicationGraphEvidence, ): JavaScriptSemanticEvidenceReference { + const fast = this[tryInternValidatedInputSymbol](evidence); + if (fast !== undefined) return fast; + const parsed = applicationGraphEvidenceSchema.parse(evidence); + return this[internValidatedEvidenceSymbol](parsed, evidence); + } + + /** Intern provenance that has already passed applicationGraphEvidenceSchema. */ + [internValidatedEvidenceSymbol]( + evidence: ApplicationGraphEvidence, + originalInput?: ApplicationGraphEvidence, + ): JavaScriptSemanticEvidenceReference { + const fast = this[tryInternValidatedInputSymbol](originalInput ?? evidence); + if (fast !== undefined) return fast; const normalized = normalizeEvidence(evidence); const { location, ...context } = normalized; const canonicalContext = canonicalJson( context, "JavaScript semantic graph", ); - const internedId = this.#idsByCanonicalContext.get(canonicalContext); - if (internedId !== undefined) return { context_id: internedId, location }; - const contextId = evidenceContextId(context); - // This canonical context is new, so a row already stored under its - // digest holds different provenance. - if (this.#contexts.has(contextId)) - throw new TypeError( - "JavaScript semantic evidence context digest collision", - ); - this.#contexts.set(contextId, { ...context, context_id: contextId }); - this.#idsByCanonicalContext.set(canonicalContext, contextId); + let contextId = this.#idsByCanonicalContext.get(canonicalContext); + if (contextId === undefined) { + contextId = evidenceContextId(context); + // This canonical context is new, so a row already stored under its + // digest holds different provenance. + if (this.#contexts.has(contextId)) + throw new TypeError( + "JavaScript semantic evidence context digest collision", + ); + const parsedContext = javaScriptSemanticEvidenceContextSchema.parse({ + ...context, + context_id: contextId, + }); + this.#contexts.set(contextId, sealSemanticEvidenceContext(parsedContext)); + this.#idsByCanonicalContext.set(canonicalContext, contextId); + } + const candidate = originalInput ?? evidence; + if ( + this.#frozenEntries.length < 16 && + hasReusableFrozenContext(candidate) && + !this.#frozenEntries.some( + (entry) => + entry.authority === candidate.authority && + entry.state === candidate.state && + entry.confidence === candidate.confidence && + entry.artifact === candidate.artifact && + entry.extractor === candidate.extractor && + entry.coverage === candidate.coverage && + entry.limitations === candidate.limitations && + entry.evidenceIds === candidate.evidence_ids, + ) + ) { + this.#frozenEntries.push({ + authority: candidate.authority, + state: candidate.state, + confidence: candidate.confidence, + artifact: candidate.artifact, + extractor: candidate.extractor, + coverage: candidate.coverage, + limitations: candidate.limitations, + evidenceIds: candidate.evidence_ids, + contextId, + }); + } return { context_id: contextId, location }; } @@ -98,8 +246,17 @@ export class JavaScriptSemanticEvidenceContextRegistry { return { ...evidenceContext, location: reference.location }; } - /** Return the deterministic context table owned by this graph projection. */ + /** Return caller-owned copies of this registry's context table. */ get contexts(): JavaScriptSemanticEvidenceContext[] { + return [...this.#contexts.values()] + .sort((left, right) => + compareUnicodeCodePoints(left.context_id, right.context_id), + ) + .map((context) => structuredClone(context)); + } + + /** Return sealed contexts for an exclusively owned graph builder. */ + get ownedContexts(): JavaScriptSemanticEvidenceContext[] { return [...this.#contexts.values()].sort((left, right) => compareUnicodeCodePoints(left.context_id, right.context_id), ); @@ -109,16 +266,25 @@ export class JavaScriptSemanticEvidenceContextRegistry { clear(): void { this.#contexts.clear(); this.#idsByCanonicalContext.clear(); + this.#frozenEntries.length = 0; } /** Validate and intern provenance decoded in this realm from an analysis worker. */ adopt(input: unknown): void { - const context = javaScriptSemanticEvidenceContextSchema.parse(input); + const context = validatedSemanticEvidenceContexts.has(input as object) + ? (input as JavaScriptSemanticEvidenceContext) + : freezeOwnedJsonTreeSnapshot( + javaScriptSemanticEvidenceContextSchema.parse(input), + ); + if (!validatedSemanticEvidenceContexts.has(context)) { + const { context_id: identifier, ...value } = context; + if (identifier !== evidenceContextId(value)) + throw new TypeError( + "Transferred semantic evidence context identifier is stale", + ); + validatedSemanticEvidenceContexts.add(context); + } const { context_id: identifier, ...value } = context; - if (identifier !== evidenceContextId(value)) - throw new TypeError( - "Transferred semantic evidence context identifier is stale", - ); const canonical = canonicalJson( value, "JavaScript semantic evidence context", @@ -139,22 +305,188 @@ export class JavaScriptSemanticEvidenceContextRegistry { } } +const validatedSemanticEvidenceContexts = new WeakSet(); +const validatedSemanticNodes = new WeakSet(); +const validatedSemanticRelations = new WeakSet(); +const validatedSemanticUnknowns = new WeakSet(); +const validatedSemanticFingerprints = new WeakSet(); + +const isValidatedSemanticNode = ( + value: unknown, +): value is JavaScriptSemanticGraphNode => + typeof value === "object" && + value !== null && + validatedSemanticNodes.has(value); +const isValidatedSemanticRelation = ( + value: unknown, +): value is JavaScriptSemanticGraphRelation => + typeof value === "object" && + value !== null && + validatedSemanticRelations.has(value); +const isValidatedSemanticUnknown = ( + value: unknown, +): value is JavaScriptSemanticGraphUnknown => + typeof value === "object" && + value !== null && + validatedSemanticUnknowns.has(value); +const isValidatedSemanticFingerprint = ( + value: unknown, +): value is JavaScriptSemanticFingerprint => + typeof value === "object" && + value !== null && + validatedSemanticFingerprints.has(value); +const isValidatedSemanticEvidenceContext = ( + value: unknown, +): value is JavaScriptSemanticEvidenceContext => + typeof value === "object" && + value !== null && + validatedSemanticEvidenceContexts.has(value); + +const sealSemanticEvidenceContext = ( + context: JavaScriptSemanticEvidenceContext, +): JavaScriptSemanticEvidenceContext => { + const sealed = freezeOwnedJsonTreeSnapshot(context); + validatedSemanticEvidenceContexts.add(sealed); + return sealed; +}; + +const freezeEvidenceLocation = ( + location: JavaScriptSemanticEvidenceReference["location"], +): JavaScriptSemanticEvidenceReference["location"] => { + if (location.available) { + if (location.value.kind === "source-range") { + Object.freeze(location.value.start); + Object.freeze(location.value.end); + } + Object.freeze(location.value); + } + return Object.freeze(location); +}; + +const freezeEvidenceReference = ( + evidence: JavaScriptSemanticEvidenceReference, +): JavaScriptSemanticEvidenceReference => { + freezeEvidenceLocation(evidence.location); + return Object.freeze(evidence); +}; + +const freezeProperties = ( + properties: JavaScriptSemanticGraphNode["properties"], +): JavaScriptSemanticGraphNode["properties"] => { + const keys = Object.keys(properties); + if (keys.length === 0) return Object.freeze(properties); + if ( + keys.every((key) => { + const item = properties[key]; + return item === null || typeof item !== "object"; + }) + ) + return Object.freeze(properties); + return freezeOwnedJsonTreeSnapshot(properties); +}; + +const sealSemanticNode = ( + node: JavaScriptSemanticGraphNode, +): JavaScriptSemanticGraphNode => { + if (node.identity.source_range !== null) { + Object.freeze(node.identity.source_range.start); + Object.freeze(node.identity.source_range.end); + Object.freeze(node.identity.source_range); + } + Object.freeze(node.identity); + Object.freeze(node.application_node_ids); + freezeProperties(node.properties); + freezeEvidenceReference(node.evidence); + const sealed = Object.freeze(node); + validatedSemanticNodes.add(sealed); + return sealed; +}; +const sealSemanticRelation = ( + relation: JavaScriptSemanticGraphRelation, +): JavaScriptSemanticGraphRelation => { + freezeProperties(relation.properties); + freezeEvidenceReference(relation.evidence); + const sealed = Object.freeze(relation); + validatedSemanticRelations.add(sealed); + return sealed; +}; +const sealSemanticUnknown = ( + unknown: JavaScriptSemanticGraphUnknown, +): JavaScriptSemanticGraphUnknown => { + Object.freeze(unknown.relation_kinds); + Object.freeze(unknown.candidate_node_ids); + freezeEvidenceReference(unknown.evidence); + const sealed = Object.freeze(unknown); + validatedSemanticUnknowns.add(sealed); + return sealed; +}; +const sealSemanticFingerprint = ( + fingerprint: JavaScriptSemanticFingerprint, +): JavaScriptSemanticFingerprint => { + Object.freeze(fingerprint.components.effects); + Object.freeze(fingerprint.components); + Object.freeze(fingerprint.limitations); + freezeEvidenceReference(fingerprint.evidence); + const sealed = Object.freeze(fingerprint); + validatedSemanticFingerprints.add(sealed); + return sealed; +}; + +const nodeInputWithoutEvidenceSchema = javaScriptSemanticNodeInputSchema.omit({ + evidence: true, +}); +const relationInputWithoutEvidenceSchema = + javaScriptSemanticRelationInputSchema.omit({ evidence: true }); +const unknownInputWithoutEvidenceSchema = + javaScriptSemanticUnknownInputSchema.omit({ evidence: true }); +const fingerprintInputWithoutEvidenceSchema = + javaScriptSemanticFingerprintInputSchema.omit({ evidence: true }); +const applicationNodeIdsSchema = z.array(prefixedDigestSchema("jag_node")); + /** Derive a semantic node ID from its kind and exact artifact identity. */ export const javaScriptSemanticNodeId = ( node: Pick, ): string => `jsrg_node_${digestCanonicalValue({ kind: node.kind, identity: node.identity }, "JavaScript semantic graph")}`; -/** Normalize one semantic entity and derive its artifact-version identifier. */ -export const createJavaScriptSemanticGraphNode = ( +const buildSemanticGraphNode = ( input: unknown, evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, ): JavaScriptSemanticGraphNode => { + if (isRecord(input) && Object.hasOwn(input, "evidence")) { + const fastEvidence = evidenceContexts[tryInternValidatedInputSymbol]( + input.evidence, + ); + if (fastEvidence !== undefined) { + const { evidence: _evidence, ...rest } = input; + const parsedRest = nodeInputWithoutEvidenceSchema.safeParse(rest); + if (parsedRest.success) { + const semantic = { + ...parsedRest.data, + application_node_ids: + parsedRest.data.application_node_ids.length <= 1 + ? parsedRest.data.application_node_ids + : uniqueSorted(parsedRest.data.application_node_ids), + evidence: fastEvidence, + }; + return { + ...semantic, + node_id: javaScriptSemanticNodeId(semantic), + }; + } + } + } + const rawEvidence = isRecord(input) + ? (input.evidence as ApplicationGraphEvidence | undefined) + : undefined; const parsed = javaScriptSemanticNodeInputSchema.parse(input); const semantic = { ...parsed, application_node_ids: uniqueSorted(parsed.application_node_ids), - evidence: evidenceContexts.intern(parsed.evidence), + evidence: evidenceContexts[internValidatedEvidenceSymbol]( + parsed.evidence, + rawEvidence, + ), }; return { ...semantic, @@ -162,15 +494,89 @@ export const createJavaScriptSemanticGraphNode = ( }; }; -/** Normalize one semantic relationship and derive its exact identifier. */ -export const createJavaScriptSemanticGraphRelation = ( +/** Normalize one semantic entity and derive its artifact-version identifier. */ +export const createJavaScriptSemanticGraphNode = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticGraphNode => + buildSemanticGraphNode(input, evidenceContexts); + +/** Normalize and seal one exclusively owned semantic entity. */ +export const createOwnedJavaScriptSemanticGraphNode = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticGraphNode => + sealSemanticNode(buildSemanticGraphNode(input, evidenceContexts)); + +/** Bind validated application-node identifiers while preserving node sealing. */ +export const withJavaScriptSemanticNodeApplicationIds = ( + node: JavaScriptSemanticGraphNode, + applicationNodeIds: readonly string[], +): JavaScriptSemanticGraphNode => { + const normalized = + applicationNodeIds.length === 0 + ? [] + : uniqueSorted(applicationNodeIdsSchema.parse(applicationNodeIds)); + if ( + validatedSemanticNodes.has(node) && + normalized.length === node.application_node_ids.length && + normalized.every((id, index) => id === node.application_node_ids[index]) + ) + return node; + if (!validatedSemanticNodes.has(node)) { + const parsed = javaScriptSemanticNodeSchema.parse({ + ...node, + application_node_ids: normalized, + }); + if (parsed.node_id !== javaScriptSemanticNodeId(parsed)) + throw new z.ZodError([ + { + code: "custom", + path: ["node_id"], + message: "Node identifier is stale", + }, + ]); + return sealSemanticNode(parsed); + } + return sealSemanticNode({ + ...node, + application_node_ids: normalized, + }); +}; + +const buildSemanticGraphRelation = ( input: unknown, evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, ): JavaScriptSemanticGraphRelation => { + if (isRecord(input) && Object.hasOwn(input, "evidence")) { + const fastEvidence = evidenceContexts[tryInternValidatedInputSymbol]( + input.evidence, + ); + if (fastEvidence !== undefined) { + const { evidence: _evidence, ...rest } = input; + const parsedRest = relationInputWithoutEvidenceSchema.safeParse(rest); + if (parsedRest.success) { + const semantic = { + ...parsedRest.data, + evidence: fastEvidence, + }; + return { + ...semantic, + relation_id: `jsrg_relation_${digestCanonicalValue(semantic, "JavaScript semantic graph")}`, + }; + } + } + } + const rawEvidence = isRecord(input) + ? (input.evidence as ApplicationGraphEvidence | undefined) + : undefined; const parsed = javaScriptSemanticRelationInputSchema.parse(input); const semantic = { ...parsed, - evidence: evidenceContexts.intern(parsed.evidence), + evidence: evidenceContexts[internValidatedEvidenceSymbol]( + parsed.evidence, + rawEvidence, + ), }; return { ...semantic, @@ -178,17 +584,57 @@ export const createJavaScriptSemanticGraphRelation = ( }; }; -/** Normalize one unresolved semantic frontier and derive its identifier. */ -export const createJavaScriptSemanticGraphUnknown = ( +/** Normalize one semantic relationship and derive its exact identifier. */ +export const createJavaScriptSemanticGraphRelation = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticGraphRelation => + buildSemanticGraphRelation(input, evidenceContexts); + +/** Normalize and seal one exclusively owned semantic relationship. */ +export const createOwnedJavaScriptSemanticGraphRelation = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticGraphRelation => + sealSemanticRelation(buildSemanticGraphRelation(input, evidenceContexts)); + +const buildSemanticGraphUnknown = ( input: unknown, evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, ): JavaScriptSemanticGraphUnknown => { + if (isRecord(input) && Object.hasOwn(input, "evidence")) { + const fastEvidence = evidenceContexts[tryInternValidatedInputSymbol]( + input.evidence, + ); + if (fastEvidence !== undefined) { + const { evidence: _evidence, ...rest } = input; + const parsedRest = unknownInputWithoutEvidenceSchema.safeParse(rest); + if (parsedRest.success) { + const semantic = { + ...parsedRest.data, + relation_kinds: uniqueSorted(parsedRest.data.relation_kinds), + candidate_node_ids: uniqueSorted(parsedRest.data.candidate_node_ids), + evidence: fastEvidence, + }; + return { + ...semantic, + unknown_id: `jsrg_unknown_${digestCanonicalValue(semantic, "JavaScript semantic graph")}`, + }; + } + } + } + const rawEvidence = isRecord(input) + ? (input.evidence as ApplicationGraphEvidence | undefined) + : undefined; const parsed = javaScriptSemanticUnknownInputSchema.parse(input); const semantic = { ...parsed, relation_kinds: uniqueSorted(parsed.relation_kinds), candidate_node_ids: uniqueSorted(parsed.candidate_node_ids), - evidence: evidenceContexts.intern(parsed.evidence), + evidence: evidenceContexts[internValidatedEvidenceSymbol]( + parsed.evidence, + rawEvidence, + ), }; return { ...semantic, @@ -196,11 +642,69 @@ export const createJavaScriptSemanticGraphUnknown = ( }; }; -/** Normalize one function fingerprint and derive its component commitment. */ -export const createJavaScriptSemanticFingerprint = ( +/** Normalize one unresolved semantic frontier and derive its identifier. */ +export const createJavaScriptSemanticGraphUnknown = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticGraphUnknown => + buildSemanticGraphUnknown(input, evidenceContexts); + +/** Normalize and seal one exclusively owned unresolved semantic frontier. */ +export const createOwnedJavaScriptSemanticGraphUnknown = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticGraphUnknown => + sealSemanticUnknown(buildSemanticGraphUnknown(input, evidenceContexts)); + +const buildSemanticFingerprint = ( input: unknown, evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, ): JavaScriptSemanticFingerprint => { + if (isRecord(input) && Object.hasOwn(input, "evidence")) { + const fastEvidence = evidenceContexts[tryInternValidatedInputSymbol]( + input.evidence, + ); + if (fastEvidence !== undefined) { + const { evidence: _evidence, ...rest } = input; + const parsedRest = fingerprintInputWithoutEvidenceSchema.safeParse(rest); + if (parsedRest.success) { + const semantic = { + ...parsedRest.data, + components: { + ...parsedRest.data.components, + effects: + parsedRest.data.components.effects.length <= 1 + ? parsedRest.data.components.effects + : uniqueSorted(parsedRest.data.components.effects), + }, + limitations: + parsedRest.data.limitations.length <= 1 + ? parsedRest.data.limitations + : uniqueSorted(parsedRest.data.limitations), + evidence: fastEvidence, + }; + const fingerprintSha256 = digestCanonicalValue( + semantic.components, + "JavaScript semantic graph", + ); + return { + ...semantic, + fingerprint_sha256: fingerprintSha256, + fingerprint_id: `jsrg_fingerprint_${digestCanonicalValue( + { + function_node_id: semantic.function_node_id, + algorithm: semantic.algorithm, + fingerprint_sha256: fingerprintSha256, + }, + "JavaScript semantic graph", + )}`, + }; + } + } + } + const rawEvidence = isRecord(input) + ? (input.evidence as ApplicationGraphEvidence | undefined) + : undefined; const parsed = javaScriptSemanticFingerprintInputSchema.parse(input); const semantic = { ...parsed, @@ -209,7 +713,10 @@ export const createJavaScriptSemanticFingerprint = ( effects: uniqueSorted(parsed.components.effects), }, limitations: uniqueSorted(parsed.limitations), - evidence: evidenceContexts.intern(parsed.evidence), + evidence: evidenceContexts[internValidatedEvidenceSymbol]( + parsed.evidence, + rawEvidence, + ), }; const fingerprintSha256 = digestCanonicalValue( semantic.components, @@ -229,6 +736,20 @@ export const createJavaScriptSemanticFingerprint = ( }; }; +/** Normalize one function fingerprint and derive its component commitment. */ +export const createJavaScriptSemanticFingerprint = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticFingerprint => + buildSemanticFingerprint(input, evidenceContexts); + +/** Normalize and seal one exclusively owned function fingerprint. */ +export const createOwnedJavaScriptSemanticFingerprint = ( + input: unknown, + evidenceContexts: JavaScriptSemanticEvidenceContextRegistry, +): JavaScriptSemanticFingerprint => + sealSemanticFingerprint(buildSemanticFingerprint(input, evidenceContexts)); + type GraphRecord = z.infer; type SemanticGraphIssue = { @@ -425,9 +946,20 @@ function* checkEvidenceContextReferencesSteps( graph: GraphRecord, context: GraphIssueReporter, ): Generator { - const contexts = new Map( - graph.evidence_contexts.map((value) => [value.context_id, value]), - ); + const contexts = new Map< + string, + { + readonly value: JavaScriptSemanticEvidenceContext; + readonly validContext: boolean; + } + >(); + for (const value of graph.evidence_contexts) { + const { context_id: _contextId, ...evidenceValue } = value; + contexts.set(value.context_id, { + value, + validContext: isValidApplicationGraphEvidenceContext(evidenceValue), + }); + } const referenced = new Set(); const validateReferences = function* ( entities: readonly { evidence: JavaScriptSemanticEvidenceReference }[], @@ -436,8 +968,8 @@ function* checkEvidenceContextReferencesSteps( for (const [index, { evidence }] of entities.entries()) { if (index % 256 === 0) yield; referenced.add(evidence.context_id); - const evidenceContext = contexts.get(evidence.context_id); - if (evidenceContext === undefined) { + const entry = contexts.get(evidence.context_id); + if (entry === undefined) { context.addIssue({ code: "custom", path: [entityKind, index, "evidence", "context_id"], @@ -445,12 +977,12 @@ function* checkEvidenceContextReferencesSteps( }); continue; } - const { context_id: _contextId, ...evidenceValue } = evidenceContext; if ( - !applicationGraphEvidenceSchema.safeParse({ - ...evidenceValue, - location: evidence.location, - }).success + !entry.validContext || + !isCompatibleApplicationGraphEvidenceLocation( + entry.value, + evidence.location, + ) ) context.addIssue({ code: "custom", @@ -466,7 +998,10 @@ function* checkEvidenceContextReferencesSteps( for (const [index, evidenceContext] of graph.evidence_contexts.entries()) { if (index % 256 === 0) yield; const { context_id: identifier, ...value } = evidenceContext; - if (identifier !== evidenceContextId(value)) + if ( + !validatedSemanticEvidenceContexts.has(evidenceContext) && + identifier !== evidenceContextId(value) + ) context.addIssue({ code: "custom", path: ["evidence_contexts", index, "context_id"], @@ -488,7 +1023,10 @@ function* checkNodesSteps( ): Generator { for (const [index, node] of graph.nodes.entries()) { if (index % 256 === 0) yield; - if (node.node_id !== javaScriptSemanticNodeId(node)) + if ( + !validatedSemanticNodes.has(node) && + node.node_id !== javaScriptSemanticNodeId(node) + ) context.addIssue({ code: "custom", path: ["nodes", index, "node_id"], @@ -530,17 +1068,19 @@ function* checkRelationsSteps( ): Generator { for (const [index, relation] of graph.relations.entries()) { if (index % 256 === 0) yield; - const { relation_id: identifier, ...semantic } = relation; - if ( - identifier !== - "jsrg_relation_" + - digestCanonicalValue(semantic, "JavaScript semantic graph") - ) - context.addIssue({ - code: "custom", - path: ["relations", index, "relation_id"], - message: "Relation identifier is stale", - }); + if (!validatedSemanticRelations.has(relation)) { + const { relation_id: identifier, ...semantic } = relation; + if ( + identifier !== + "jsrg_relation_" + + digestCanonicalValue(semantic, "JavaScript semantic graph") + ) + context.addIssue({ + code: "custom", + path: ["relations", index, "relation_id"], + message: "Relation identifier is stale", + }); + } if ( !nodes.has(relation.source_node_id) || !nodes.has(relation.target_node_id) @@ -576,17 +1116,19 @@ function* checkUnknownsSteps( ): Generator { for (const [index, unknown] of graph.unknowns.entries()) { if (index % 256 === 0) yield; - const { unknown_id: identifier, ...semantic } = unknown; - if ( - identifier !== - "jsrg_unknown_" + - digestCanonicalValue(semantic, "JavaScript semantic graph") - ) - context.addIssue({ - code: "custom", - path: ["unknowns", index, "unknown_id"], - message: "Unknown frontier identifier is stale", - }); + if (!validatedSemanticUnknowns.has(unknown)) { + const { unknown_id: identifier, ...semantic } = unknown; + if ( + identifier !== + "jsrg_unknown_" + + digestCanonicalValue(semantic, "JavaScript semantic graph") + ) + context.addIssue({ + code: "custom", + path: ["unknowns", index, "unknown_id"], + message: "Unknown frontier identifier is stale", + }); + } if (unknown.node_id !== null && !nodes.has(unknown.node_id)) context.addIssue({ code: "custom", @@ -650,31 +1192,36 @@ function* checkFingerprintsSteps( path: ["fingerprints", index, "function_node_id"], message: "Fingerprint must name a function node", }); - if ( - fingerprint.fingerprint_sha256 !== - digestCanonicalValue(fingerprint.components, "JavaScript semantic graph") - ) - context.addIssue({ - code: "custom", - path: ["fingerprints", index, "fingerprint_sha256"], - message: "Fingerprint component commitment is stale", - }); - const expectedIdentifier = - "jsrg_fingerprint_" + - digestCanonicalValue( - { - function_node_id: fingerprint.function_node_id, - algorithm: fingerprint.algorithm, - fingerprint_sha256: fingerprint.fingerprint_sha256, - }, - "JavaScript semantic graph", - ); - if (fingerprint.fingerprint_id !== expectedIdentifier) - context.addIssue({ - code: "custom", - path: ["fingerprints", index, "fingerprint_id"], - message: "Fingerprint identifier is stale", - }); + if (!validatedSemanticFingerprints.has(fingerprint)) { + if ( + fingerprint.fingerprint_sha256 !== + digestCanonicalValue( + fingerprint.components, + "JavaScript semantic graph", + ) + ) + context.addIssue({ + code: "custom", + path: ["fingerprints", index, "fingerprint_sha256"], + message: "Fingerprint component commitment is stale", + }); + const expectedIdentifier = + "jsrg_fingerprint_" + + digestCanonicalValue( + { + function_node_id: fingerprint.function_node_id, + algorithm: fingerprint.algorithm, + fingerprint_sha256: fingerprint.fingerprint_sha256, + }, + "JavaScript semantic graph", + ); + if (fingerprint.fingerprint_id !== expectedIdentifier) + context.addIssue({ + code: "custom", + path: ["fingerprints", index, "fingerprint_id"], + message: "Fingerprint identifier is stale", + }); + } if ( fingerprint.status !== "complete" && fingerprint.limitations.length === 0 @@ -804,14 +1351,72 @@ const normalizeGraphInput = ( limitations: uniqueSorted(parsed.limitations), }); +const parseElementsSync = ( + element: Element, + values: readonly unknown[], + path: readonly PropertyKey[], + issues: z.core.$ZodIssue[], + isPrevalidated?: (value: unknown) => value is z.output, +): z.output[] => { + const steps = parseElementSteps( + element, + values, + path, + issues, + isPrevalidated, + ); + let step = steps.next(); + while (!step.done) step = steps.next(); + return step.value; +}; + /** Normalize a complete companion graph and derive its graph ID. */ export const createJavaScriptSemanticGraph = ( input: unknown, ): JavaScriptSemanticGraph => { - const semantic = normalizeGraphInput( - javaScriptSemanticGraphInputSchema.parse(input), - ); - // The input schema already cloned and validated every field. Validate the + const container = semanticGraphContainerSchema.parse(input); + const parseIssues: z.core.$ZodIssue[] = []; + const parsed: JavaScriptSemanticGraphInput = { + ...container, + evidence_contexts: parseElementsSync( + semanticGraphShape.evidence_contexts.element, + container.evidence_contexts, + ["evidence_contexts"], + parseIssues, + isValidatedSemanticEvidenceContext, + ), + nodes: parseElementsSync( + semanticGraphShape.nodes.element, + container.nodes, + ["nodes"], + parseIssues, + isValidatedSemanticNode, + ), + relations: parseElementsSync( + semanticGraphShape.relations.element, + container.relations, + ["relations"], + parseIssues, + isValidatedSemanticRelation, + ), + fingerprints: parseElementsSync( + semanticGraphShape.fingerprints.element, + container.fingerprints, + ["fingerprints"], + parseIssues, + isValidatedSemanticFingerprint, + ), + unknowns: parseElementsSync( + semanticGraphShape.unknowns.element, + container.unknowns, + ["unknowns"], + parseIssues, + isValidatedSemanticUnknown, + ), + }; + if (parseIssues.length > 0) throw new z.ZodError(parseIssues); + const semantic = normalizeGraphInput(parsed); + // The input schema or factory sealing already validated every field. Validate // normalized relationships in place, then derive the only added field. The // public schema still verifies arbitrary records and their commitments. const record: GraphRecord = { @@ -861,30 +1466,35 @@ function* parseAndSealSemanticGraphSteps( container.evidence_contexts, ["evidence_contexts"], issues, + isValidatedSemanticEvidenceContext, ), nodes: yield* parseElementSteps( semanticGraphShape.nodes.element, container.nodes, ["nodes"], issues, + isValidatedSemanticNode, ), relations: yield* parseElementSteps( semanticGraphShape.relations.element, container.relations, ["relations"], issues, + isValidatedSemanticRelation, ), fingerprints: yield* parseElementSteps( semanticGraphShape.fingerprints.element, container.fingerprints, ["fingerprints"], issues, + isValidatedSemanticFingerprint, ), unknowns: yield* parseElementSteps( semanticGraphShape.unknowns.element, container.unknowns, ["unknowns"], issues, + isValidatedSemanticUnknown, ), }; if (issues.length > 0) throw new z.ZodError(issues); diff --git a/src/domain/javascript/javascriptSemanticGraphQuery.test.ts b/src/domain/javascript/javascriptSemanticGraphQuery.test.ts index 492da10c72..3f1b9c8ec7 100644 --- a/src/domain/javascript/javascriptSemanticGraphQuery.test.ts +++ b/src/domain/javascript/javascriptSemanticGraphQuery.test.ts @@ -25,6 +25,9 @@ import { createJavaScriptSemanticGraphNode, createJavaScriptSemanticGraphRelation, createJavaScriptSemanticGraphUnknown, + createOwnedJavaScriptSemanticFingerprint, + createOwnedJavaScriptSemanticGraphNode, + withJavaScriptSemanticNodeApplicationIds, sealTransferredJavaScriptSemanticGraphSteps, isValidatedImmutableJavaScriptSemanticGraph, JavaScriptSemanticEvidenceContextRegistry, @@ -915,3 +918,162 @@ it("interns each distinct evidence context once under its content digest", () => ); expect(registry.resolve(repeated)).toEqual(elsewhere); }); + +it("rejects invalid provenance in direct registry.intern calls before sealing", () => { + const registry = new JavaScriptSemanticEvidenceContextRegistry(); + const baseEvidence = semanticGraphTestEvidence(); + expect(() => + registry.intern({ + ...baseEvidence, + extractor: { ...baseEvidence.extractor, name: "" }, + }), + ).toThrow(z.ZodError); + expect(registry.ownedContexts).toEqual([]); +}); + +it("seals owned builder records while still rejecting tampered spread copies", () => { + const registry = new JavaScriptSemanticEvidenceContextRegistry(); + const ownedModule = createOwnedJavaScriptSemanticGraphNode( + { + kind: "module", + identity: { + artifact_sha256: SHA, + module_path: "bundle.js", + source_range: { + start: { line: 1, column: 0 }, + end: { line: 1, column: 20 }, + }, + role_key: "module", + }, + function_node_id: null, + application_node_ids: [], + label: "bundle.js", + properties: {}, + evidence: semanticGraphTestEvidence(), + }, + registry, + ); + const ownedCallable = withJavaScriptSemanticNodeApplicationIds( + createOwnedJavaScriptSemanticGraphNode( + { + kind: "function", + identity: { + artifact_sha256: SHA, + module_path: "bundle.js", + source_range: { + start: { line: 1, column: 2 }, + end: { line: 1, column: 18 }, + }, + role_key: "fn", + }, + function_node_id: null, + application_node_ids: [], + label: "fn", + properties: {}, + evidence: semanticGraphTestEvidence(), + }, + registry, + ), + [`jag_node_${"c".repeat(64)}`], + ); + const ownedFingerprint = createOwnedJavaScriptSemanticFingerprint( + { + function_node_id: ownedCallable.node_id, + algorithm: "rea.javascript-semantic-function", + status: "complete", + components: { + parameter_arity: 0, + normalized_ast_sha256: "1".repeat(64), + control_flow_sha256: "2".repeat(64), + relation_shape_sha256: "3".repeat(64), + literal_set_sha256: "4".repeat(64), + effects: ["network"], + }, + limitations: [], + evidence: semanticGraphTestEvidence("inferred"), + }, + registry, + ); + expect(Object.isFrozen(ownedModule)).toBe(true); + expect(Object.isFrozen(ownedCallable)).toBe(true); + expect(Object.isFrozen(ownedCallable.application_node_ids)).toBe(true); + expect(Object.isFrozen(ownedFingerprint)).toBe(true); + expect(Object.isFrozen(registry.ownedContexts[0])).toBe(true); + + const ownedInput: JavaScriptSemanticGraphInput = { + schema: "JavaScriptSemanticRelationGraph", + root_artifact_sha256: SHA, + application_graph_id: `jag_${"b".repeat(64)}`, + root_node_ids: [ownedModule.node_id], + evidence_contexts: registry.ownedContexts, + nodes: [ownedModule, ownedCallable], + relations: [], + unknowns: [], + fingerprints: [ownedFingerprint], + coverage: { + status: "complete", + truncated: false, + omitted_nodes: 0, + omitted_relations: 0, + limits: [], + families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ + family, + status: "complete", + retained_relations: 0, + omitted_relations: 0, + unknown_ids: [], + })), + }, + limitations: [], + }; + const direct = createJavaScriptSemanticGraph(ownedInput); + const transferred = complete( + sealTransferredJavaScriptSemanticGraphSteps(ownedInput), + ); + const unsealed = createJavaScriptSemanticGraph( + JSON.parse(JSON.stringify(ownedInput)) as JavaScriptSemanticGraphInput, + ); + expect(direct.graph_id).toBe(unsealed.graph_id); + expect(transferred).toEqual(direct); + + for (const tampered of [ + { + ...ownedInput, + nodes: [ + ownedModule, + { ...ownedCallable, node_id: `jsrg_node_${"f".repeat(64)}` }, + ], + }, + { + ...ownedInput, + fingerprints: [ + { + ...ownedFingerprint, + fingerprint_id: `jsrg_fp_${"f".repeat(64)}`, + }, + ], + }, + { + ...ownedInput, + nodes: [ + { + ...ownedModule, + evidence: { + context_id: ownedModule.evidence.context_id, + location: { + available: false as const, + reason: "unresolved" as const, + detail: "Contradicts observed context.", + }, + }, + }, + ownedCallable, + ], + }, + ] satisfies JavaScriptSemanticGraphInput[]) { + expect(() => createJavaScriptSemanticGraph(tampered)).toThrow(z.ZodError); + expect(() => + complete(sealTransferredJavaScriptSemanticGraphSteps(tampered)), + ).toThrow(z.ZodError); + } +}); diff --git a/src/domain/zodCollectionSteps.ts b/src/domain/zodCollectionSteps.ts index a32e8c8044..33ff5b3eb0 100644 --- a/src/domain/zodCollectionSteps.ts +++ b/src/domain/zodCollectionSteps.ts @@ -15,10 +15,15 @@ export function* parseElementSteps( values: readonly unknown[], path: readonly PropertyKey[], issues: z.core.$ZodIssue[], + isPrevalidated?: (value: unknown) => value is z.output, ): Generator[]> { const parsed: z.output[] = []; for (const [index, value] of values.entries()) { if (index % 64 === 0) yield; + if (isPrevalidated?.(value) === true) { + parsed.push(value); + continue; + } const result = element.safeParse(value); if (result.success) parsed.push(result.data); else diff --git a/src/reference/ReferenceSourceReaderValidate.ts b/src/reference/ReferenceSourceReaderValidate.ts index 63a08f2e40..410e4ebd4f 100644 --- a/src/reference/ReferenceSourceReaderValidate.ts +++ b/src/reference/ReferenceSourceReaderValidate.ts @@ -57,7 +57,7 @@ export const validateDirectory = async ( const rootNow = await bigLstat(root); if (!sameFile(rootIdentity, rootNow)) return { ok: false, code: "changed", message: "Reference root changed" }; - const stats = await bigLstat(path); + const stats = path === root ? rootNow : await bigLstat(path); if (stats.isSymbolicLink()) return { ok: false, diff --git a/src/resources/DirectoryArtifactReader.ts b/src/resources/DirectoryArtifactReader.ts index 859d8c7db1..78d5337a24 100644 --- a/src/resources/DirectoryArtifactReader.ts +++ b/src/resources/DirectoryArtifactReader.ts @@ -262,11 +262,11 @@ export class DirectoryArtifactReader implements ArtifactReader { try { for (;;) { abortIfNeeded(signal); + const child = await handle.handle.read(); + if (child === null) break; await withDirectoryIoContext("inspect directory", directory, () => assertContainedDirectory(root, directory, admittedRoot.identity), ); - const child = await handle.handle.read(); - if (child === null) break; examine(); abortIfNeeded(signal); const decoded = decodeDirectoryEntryName(child.name); @@ -759,10 +759,29 @@ const assertContainedDirectory = async ( rootIdentity: { readonly dev: bigint; readonly ino: bigint } | undefined, failureReason: "path" | "integrity" = "path", ): Promise => { - const canonical = await realpath(directory); + const [canonicalResult, metadataResult, currentRootResult] = + directory === root + ? await Promise.allSettled([ + realpath(root), + lstat(root, { bigint: true }), + ]) + : await Promise.allSettled([ + realpath(directory), + lstat(directory, { bigint: true }), + lstat(root, { bigint: true }), + ]); + if (canonicalResult.status === "rejected") throw canonicalResult.reason; + if (metadataResult.status === "rejected") throw metadataResult.reason; + if ( + currentRootResult !== undefined && + currentRootResult.status === "rejected" + ) + throw currentRootResult.reason; + const canonical = canonicalResult.value; const fromRoot = relative(root, canonical); - const metadata = await lstat(directory); - const currentRoot = await lstat(root, { bigint: true }); + const metadata = metadataResult.value; + const currentRoot = + currentRootResult === undefined ? metadata : currentRootResult.value; if ( rootIdentity === undefined || currentRoot.dev !== rootIdentity.dev || diff --git a/tests/boundary/cli/cliParityGoldens.test.ts b/tests/boundary/cli/cliParityGoldens.test.ts index da162ee4e7..3bf589a963 100644 --- a/tests/boundary/cli/cliParityGoldens.test.ts +++ b/tests/boundary/cli/cliParityGoldens.test.ts @@ -554,7 +554,6 @@ describe("CLI parity goldens", () => { ? {} : { COMPLETE: testCase.complete }), }, - ...(testCase.complete === undefined ? {} : { freshProcess: true }), ...(testCase.terminal === true ? { terminal: true } : {}), }); const actual = render(testCase, workspace, result); diff --git a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts index a726e86fb0..d4ce2401f8 100644 --- a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts +++ b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts @@ -1,11 +1,25 @@ -import { mkdir, rename, writeFile, type FileHandle } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import type { Dirent } from "node:fs"; +import { + mkdir, + rename, + rm, + symlink, + writeFile, + type FileHandle, +} from "node:fs/promises"; import { join } from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; const controls = vi.hoisted(() => ({ wrapDirectory: undefined as - | ((handle: { close(): Promise; readonly path: string }) => void) + | ((handle: { + close(): Promise; + read(): Promise; + readonly path: string; + }) => void) | undefined, wrapFile: undefined as | ((handle: { close(): Promise; readonly fd: number }) => void) @@ -247,3 +261,68 @@ describe("DirectoryArtifactReader file handle ownership", () => { } }); }); + +it.skipIf(process.platform === "win32")( + "revalidates directory containment after a skipped special entry before inspecting the next entry", + async () => { + const parent = await createTestTempDirectory( + "rea-directory-skipped-escape-", + ); + const root = join(parent, "selected"); + const outside = join(parent, "outside"); + const sub = join(root, "sub"); + await mkdir(sub, { recursive: true }); + await mkdir(outside, { recursive: true }); + await promisify(execFile)("mkfifo", [join(sub, "a.pipe")], { + timeout: 2_000, + }); + await writeFile(join(sub, "z.txt"), "inside\n"); + await writeFile(join(outside, "z.txt"), "escaped\n"); + + controls.wrapDirectory = (handle) => { + if (!handle.path.endsWith("/sub")) return; + const read = handle.read.bind(handle); + const queued: Dirent[] = []; + let loaded = false; + let reads = 0; + handle.read = async () => { + if (!loaded) { + for (;;) { + const entry = await read(); + if (entry === null) break; + queued.push(entry); + } + queued.sort((left, right) => + String(left.name).localeCompare(String(right.name)), + ); + loaded = true; + } + const next = queued[reads] ?? null; + reads += 1; + if (reads === 2) { + await rm(sub, { recursive: true, force: true }); + await symlink(outside, sub, "dir"); + } + return next; + }; + }; + + const omissions: string[] = []; + const reader = new DirectoryArtifactReader(root); + try { + await expect(async () => { + for await (const entry of reader.entries(undefined, { + onOmission: ({ path }) => omissions.push(path), + })) { + expect(entry.path).not.toBe("sub/z.txt"); + } + }).rejects.toMatchObject({ + reason: "path", + message: "Directory traversal escaped or changed its artifact root", + }); + expect(omissions).toEqual(["sub/a.pipe"]); + } finally { + await reader.close(); + } + }, +); diff --git a/tests/boundary/filesystem/wideDirectoryTraversal.test.ts b/tests/boundary/filesystem/wideDirectoryTraversal.test.ts index e06320ede2..306b01574e 100644 --- a/tests/boundary/filesystem/wideDirectoryTraversal.test.ts +++ b/tests/boundary/filesystem/wideDirectoryTraversal.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -53,7 +53,15 @@ describe("directory traversal of a parent wider than call arguments allow", () = beforeAll(async () => { workspace = await createTestWorkspace("rea-wide-directory-"); root = await workspace.mkdir("tree"); - for (const name of childNames()) mkdirSync(join(root, name)); + const names = childNames(); + const batchSize = 64; + for (let offset = 0; offset < names.length; offset += batchSize) { + await Promise.all( + names + .slice(offset, offset + batchSize) + .map((name) => mkdir(join(root, name))), + ); + } }); afterAll(async () => { if (workspace !== undefined) await removeTestWorkspace(workspace.root); diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index 0d1cbf28ea..5ebfd0ce5a 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -495,14 +495,10 @@ describe("production MCP doctor output", () => { TMPDIR: directories.temp, }; - const json = await captureProductionCli( - ["mcp", "doctor", "--json"], - environment, - ); - const jsonl = await captureProductionCli( - ["mcp", "doctor", "--format", "jsonl"], - environment, - ); + const [json, jsonl] = await Promise.all([ + captureProductionCli(["mcp", "doctor", "--json"], environment), + captureProductionCli(["mcp", "doctor", "--format", "jsonl"], environment), + ]); expect([json.exitCode, jsonl.exitCode]).toEqual([1, 1]); const parseable = [json.stdout, jsonl.stdout].map((output) => { diff --git a/tests/boundary/providers/ida/verifierObservations.test.ts b/tests/boundary/providers/ida/verifierObservations.test.ts index dcb2c64e5b..d2408d32e2 100644 --- a/tests/boundary/providers/ida/verifierObservations.test.ts +++ b/tests/boundary/providers/ida/verifierObservations.test.ts @@ -1,8 +1,9 @@ import { randomUUID } from "node:crypto"; -import { readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; +import { tmpdir } from "node:os"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { afterAll, beforeAll, expect, it } from "vitest"; import { z } from "zod"; import { jsonValueSchema, @@ -13,8 +14,46 @@ import { OwnedCommandFailure, runOwnedCommand, } from "../../../../src/process/OwnedCommand.js"; +import type { ProviderProcessSupervisor } from "../../../../src/process/ProviderProcess.js"; import { createIdaTarget } from "../../../fixtures/idaMcp.js"; +let sharedTargetRoot = ""; +let sharedCompileCache = ""; +let sharedTarget: Awaited>["target"]; +const unresolvedCleanupOwners: ProviderProcessSupervisor[] = []; +let unownedUnconfirmedCleanups = 0; + +beforeAll(async () => { + const created = await createIdaTarget(); + sharedTargetRoot = created.root; + sharedTarget = created.target; + sharedCompileCache = join(sharedTargetRoot, "node-compile"); +}); + +afterAll(async () => { + const remainingFailures: string[] = []; + for (const owner of unresolvedCleanupOwners.splice(0)) { + const result = await owner.stop(); + if (result.status === "incomplete" || result.status === "not-owned") { + remainingFailures.push(result.status); + } else { + owner.dispose(); + } + } + if ( + sharedTargetRoot !== "" && + unownedUnconfirmedCleanups === 0 && + remainingFailures.length === 0 + ) { + await rm(sharedTargetRoot, { recursive: true, force: true }); + } + if (remainingFailures.length > 0) { + throw new Error( + `Retained IDA verifier shared root (${sharedTargetRoot}); unresolved process cleanup: ${remainingFailures.join(", ")}`, + ); + } +}); + const requestSchema = z.object({ id: z.union([z.string(), z.number()]).optional(), method: z.string(), @@ -109,7 +148,8 @@ const observation = ( }; const fixture = async (empty: readonly string[], omitMore: boolean) => { - const { root, target } = await createIdaTarget(); + const root = await mkdtemp(join(tmpdir(), "rea-ida-verifier-")); + const target = sharedTarget; const server = createServer(async (request, response) => { try { let body = ""; @@ -241,6 +281,10 @@ it.each([ env: { REA_IDA_MCP_CONFIG: upstream.config, NODE_OPTIONS: "--max-old-space-size=1024", + ...(process.env.NODE_V8_COVERAGE === undefined && + sharedCompileCache !== "" + ? { NODE_COMPILE_CACHE: sharedCompileCache } + : {}), }, }, { timeoutMs: 90000, diagnosticBytes: 16384, acceptNonZeroExit: true }, @@ -298,8 +342,17 @@ it.each([ ]); else expect(report.unverified).toEqual([]); } catch (cause) { - if (cause instanceof OwnedCommandFailure && cause.cleanupFailure === null) - cleanupConfirmed = true; + if (cause instanceof OwnedCommandFailure) { + if (cause.cleanupFailure === null) { + cleanupConfirmed = true; + } else if (cause.cleanupOwner !== undefined) { + unresolvedCleanupOwners.push(cause.cleanupOwner); + } else { + unownedUnconfirmedCleanups += 1; + } + } else if (!cleanupConfirmed) { + unownedUnconfirmedCleanups += 1; + } throw cause; } finally { await upstream.close(cleanupConfirmed); diff --git a/tests/boundary/providers/kuna/kunaVerifier.test.ts b/tests/boundary/providers/kuna/kunaVerifier.test.ts index 3c9a5ff430..b5c3ae100d 100644 --- a/tests/boundary/providers/kuna/kunaVerifier.test.ts +++ b/tests/boundary/providers/kuna/kunaVerifier.test.ts @@ -4,15 +4,72 @@ import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; -import { expect, it as test } from "vitest"; +import { afterAll, beforeAll, expect, it as test } from "vitest"; import { KUNA_VERIFIED_VERSION } from "../../../../src/kuna/KunaRelease.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../../support/workspace/workspaceFixture.js"; import { fakeKunaCalls, writeFakeKuna } from "./fakeKuna.js"; const it = test.skipIf(process.platform === "win32"); +let compileCacheWorkspace = ""; +let compileCache = ""; +let retainCompileCache = false; + +beforeAll(async () => { + if (process.platform === "win32") return; + compileCacheWorkspace = ( + await createTestWorkspace("rea-kuna-verifier-compile-") + ).root; + compileCache = join(compileCacheWorkspace, "node-compile"); +}); + +afterAll(async () => { + if (compileCacheWorkspace !== "" && !retainCompileCache) { + await removeTestWorkspace(compileCacheWorkspace); + } +}); + +const compileCacheEnv = (): Record => + compileCache === "" || process.env.NODE_V8_COVERAGE !== undefined + ? {} + : { NODE_COMPILE_CACHE: compileCache }; + +const spawnVerifierSync = ( + args: readonly string[], + options: Parameters[2] & { readonly encoding: "utf8" }, +) => { + const result = spawnSync(process.execPath, args, options); + if (result.error !== undefined || result.signal !== null) { + retainCompileCache = true; + return result; + } + try { + const parsed = JSON.parse(result.stdout) as { + verifier_run?: { + process_lineage?: { status?: string; descendants?: unknown[] }; + }; + }; + const lineage = parsed.verifier_run?.process_lineage; + if ( + lineage !== undefined && + (lineage.status !== "verified" || + !Array.isArray(lineage.descendants) || + lineage.descendants.length > 0) + ) { + retainCompileCache = true; + } + } catch { + retainCompileCache = true; + } + return result; +}; + const createCallerMatrix = async (root: string) => { const targets = join(root, "targets"); const binaries: string[] = []; @@ -79,8 +136,7 @@ it("accepts requested functions distributed across caller targets", async () => ); const { targets } = await createCallerMatrix(root); const kuna = await writeFakeKuna(root); - const result = spawnSync( - process.execPath, + const result = spawnVerifierSync( [ "scripts/verify-real-kuna.mjs", "--targets", @@ -94,6 +150,7 @@ it("accepts requested functions distributed across caller targets", async () => // also covers startup, symbol selection and owned cleanup/reporting. timeout: 30_000, env: { + ...compileCacheEnv(), PATH: process.env.PATH, HOME: root, REA_KUNA_COMMAND: kuna, @@ -262,8 +319,7 @@ it.each([ mode: 0o755, }, ); - const result = spawnSync( - process.execPath, + const result = spawnVerifierSync( [ "scripts/verify-real-kuna.mjs", "--targets", @@ -276,6 +332,7 @@ it.each([ encoding: "utf8", timeout: 10_000, env: { + ...compileCacheEnv(), PATH: process.env.PATH, HOME: root, REA_KUNA_COMMAND: kuna, @@ -319,8 +376,7 @@ it("retains examined caller targets when a later symbol inspection fails", async { mode: 0o755 }, ); const reportPath = join(root, "report.json"); - const result = spawnSync( - process.execPath, + const result = spawnVerifierSync( [ "scripts/verify-real-kuna.mjs", "--targets", @@ -334,6 +390,7 @@ it("retains examined caller targets when a later symbol inspection fails", async encoding: "utf8", timeout: 10_000, env: { + ...compileCacheEnv(), PATH: process.env.PATH, HOME: root, REA_KUNA_COMMAND: kuna, @@ -490,8 +547,7 @@ await exited; `, ); const reportPath = join(root, "report.json"); - const result = spawnSync( - process.execPath, + const result = spawnVerifierSync( [ "scripts/verify-real-kuna.mjs", "--entrypoint", @@ -503,6 +559,7 @@ await exited; encoding: "utf8", timeout: 25_000, env: { + ...compileCacheEnv(), PATH: process.env.PATH, HOME: root, REA_KUNA_COMMAND: kuna, diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts index 8874e7dfed..8a0617eaa4 100644 --- a/tests/evaluation/ciTestSelection.test.ts +++ b/tests/evaluation/ciTestSelection.test.ts @@ -208,6 +208,35 @@ it("traces shared script modules to their tests and falls back for other consume "tests/boundary/filesystem/shared.test.ts", "tests/boundary/filesystem/verifier.test.ts", ]); + // Shared fixture modules under scripts/fixtures/ trace to their importing + // tests and verifiers, while failing closed when a package/build script loads them. + const fixtureSources = { + ...sources, + "scripts/fixtures/managed/pe.mjs": "", + "scripts/verify/managed/conformance.mjs": + 'import "../../fixtures/managed/pe.mjs";', + "tests/boundary/providers/dotnet/pe.test.ts": + 'import "../../../../scripts/fixtures/managed/pe.mjs";', + "tests/boundary/providers/dotnet/conformance.test.ts": + 'const v = "scripts/verify/managed/conformance.mjs";', + }; + expect( + selectedFiles(plan(["scripts/fixtures/managed/pe.mjs"], fixtureSources)), + ).toEqual([ + "tests/boundary/providers/dotnet/conformance.test.ts", + "tests/boundary/providers/dotnet/pe.test.ts", + ]); + expect( + plan(["scripts/fixtures/managed/pe.mjs"], { + ...fixtureSources, + "package.json": + '{"scripts":{"fixtures:pe":"node scripts/fixtures/managed/pe.mjs"}}', + }), + ).toMatchObject({ + mode: "full", + reason: + "Script consumer is not traced: package.json uses scripts/fixtures/managed/pe.mjs", + }); // A build script, configuration or package script that loads it is untraced. expect( plan(["scripts/lib/shared.mjs"], {