From 218c13160ec0ed7116d6633fc76559ec39dae4d7 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:07:12 +0800 Subject: [PATCH] fix: report malformed application evidence as input errors --- .../ReconstructionObligationCandidates.ts | 9 +- .../JavaScriptApplicationEvidenceGraph.ts | 300 ++++++++++++++---- .../JavaScriptApplicationWorkflowService.ts | 132 ++++---- .../JavaScriptRuntimeReconciliationService.ts | 1 - .../JavaScriptSemanticTraceService.ts | 36 ++- src/domain/evidence.ts | 20 +- src/domain/evidenceBundle.ts | 15 +- .../filesystem/evidenceBundleFiles.test.ts | 2 +- .../javascriptApplicationWorkflows.test.ts | 11 +- .../javascriptExportContainerCoverage.test.ts | 4 +- .../javascriptExportShapeComparison.test.ts | 4 +- .../javascriptRequireLiteralPaths.test.ts | 4 +- .../filesystem/sourceToBundlePaths.test.ts | 17 +- .../javascriptSemanticTraceEvidence.test.ts | 221 +++++++++++++ .../processComparisonDiagnosticsMcp.test.ts | 2 +- 15 files changed, 613 insertions(+), 165 deletions(-) create mode 100644 tests/boundary/javascriptSemanticTraceEvidence.test.ts diff --git a/src/application/ReconstructionObligationCandidates.ts b/src/application/ReconstructionObligationCandidates.ts index b01f88d7e7..e099527698 100644 --- a/src/application/ReconstructionObligationCandidates.ts +++ b/src/application/ReconstructionObligationCandidates.ts @@ -76,7 +76,14 @@ const deriveApplicationCandidates = ( ) return; try { - const source = parseApplicationGraphEvidence(evidence); + const parsed = parseApplicationGraphEvidence(evidence); + if (!parsed.ok) { + limitations.add( + `Application Evidence ${evidence.evidence_id} could not generate candidates because it failed graph validation.`, + ); + return; + } + const source = parsed.value; for (const node of source.graph.nodes) { const candidatePolicy = applicationObligationPolicy(node.kind); if (candidatePolicy === undefined) { diff --git a/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts b/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts index 459c226acb..3a42ce32e4 100644 --- a/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts +++ b/src/application/javascript/JavaScriptApplicationEvidenceGraph.ts @@ -8,6 +8,13 @@ import { parseEvidence, type Evidence, } from "../../domain/evidence.js"; +import { + type AnalysisInputIssue, + AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; +import { projectInputIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { z } from "zod"; import { analyzeJavaScriptApplicationInputSchema, javascriptApplicationAnalysisResultSchema, @@ -30,6 +37,24 @@ export interface ApplicationGraphEvidenceSource { readonly semanticGraph: JavaScriptSemanticGraph | null; } +/** Caller-owned Evidence failed validation at the application graph boundary. */ +export interface ApplicationGraphEvidenceInputFailure { + readonly issues: readonly AnalysisInputIssue[]; +} + +/** Convert typed parser issues into this workflow's caller-facing input error. */ +export const applicationGraphEvidenceInputError = ( + operation: string, + failure: ApplicationGraphEvidenceInputFailure, +): AnalysisInputError => + new AnalysisInputError(operation, undefined, failure.issues); + +class InvalidApplicationGraphEvidence extends Error { + constructor(readonly issues: readonly AnalysisInputIssue[]) { + super("Application graph Evidence failed input validation"); + } +} + const ownedApplicationSources = new WeakMap< Evidence, ApplicationGraphEvidenceSource @@ -51,7 +76,7 @@ export const rememberOwnedApplicationGraphEvidence = ( ); ownedApplicationSources.set( evidence, - Object.freeze(staticApplicationSourceForResult(evidence, result)), + Object.freeze(staticApplicationSourceForResult(evidence, result, [])), ); return evidence; }; @@ -59,62 +84,79 @@ export const rememberOwnedApplicationGraphEvidence = ( /** Parse and authenticate one REA-produced JavaScript Application Graph Evidence. */ export const parseApplicationGraphEvidence = ( input: unknown, -): ApplicationGraphEvidenceSource => { - const evidence = parseEvidence(input); - const owned = ownedApplicationSources.get(evidence); - if (owned !== undefined) return owned; - if ( - evidence.operation === "analyze_javascript_application" && - evidence.predicate_type === "rea.javascript-application-analysis" && - providerMatches(evidence, JAVASCRIPT_APPLICATION_PROVIDER) - ) - return staticApplicationSource(evidence); - if ( - evidence.operation === "reconcile_javascript_runtime" && - evidence.predicate_type === "rea.javascript-runtime-reconciliation" && - providerMatches(evidence, JAVASCRIPT_RUNTIME_RECONCILIATION_PROVIDER) - ) - return staticRuntimeSource(evidence); - if ( - evidence.operation === "project_managed_application_graph" && - evidence.predicate_type === "rea.managed-application-graph" && - providerMatches(evidence, MANAGED_WORKFLOW_PROVIDER) - ) - return managedApplicationSource(evidence); - throw new TypeError( - "Application workflow requires authenticated analyze_javascript_application, reconcile_javascript_runtime, or project_managed_application_graph Evidence", - ); -}; + path: readonly (string | number)[] = [], +): Result< + ApplicationGraphEvidenceSource, + ApplicationGraphEvidenceInputFailure +> => + captureInputValidation(() => { + const evidence = parseEvidenceInput(input, path); + const owned = ownedApplicationSources.get(evidence); + if (owned !== undefined) return owned; + if (evidence.operation === "analyze_javascript_application") { + requirePredicate(evidence, "rea.javascript-application-analysis", path); + requireProvider(evidence, JAVASCRIPT_APPLICATION_PROVIDER, path); + return staticApplicationSource(evidence, path); + } + if (evidence.operation === "reconcile_javascript_runtime") { + requirePredicate(evidence, "rea.javascript-runtime-reconciliation", path); + requireProvider( + evidence, + JAVASCRIPT_RUNTIME_RECONCILIATION_PROVIDER, + path, + ); + return staticRuntimeSource(evidence, path); + } + if (evidence.operation === "project_managed_application_graph") { + requirePredicate(evidence, "rea.managed-application-graph", path); + requireProvider(evidence, MANAGED_WORKFLOW_PROVIDER, path); + return managedApplicationSource(evidence, path); + } + throw invalidEvidence( + path, + "operation", + "Expected authenticated JavaScript analysis, runtime reconciliation, or managed application graph Evidence.", + ); + }); const staticApplicationSource = ( evidence: Evidence, + path: readonly (string | number)[], ): ApplicationGraphEvidenceSource => { - const result = javascriptApplicationAnalysisResultSchema.parse( + const result = parseCallerValue( + (input) => javascriptApplicationAnalysisResultSchema.parse(input), evidence.normalized_result, + [...path, "normalized_result"], ); - return staticApplicationSourceForResult(evidence, result); + return staticApplicationSourceForResult(evidence, result, path); }; const staticApplicationSourceForResult = ( evidence: Evidence, result: JavaScriptApplicationAnalysisResult, + path: readonly (string | number)[], ): ApplicationGraphEvidenceSource => { if ( evidence.authority !== "shipped-artifact" || evidence.confidence !== "derived" ) - throw new TypeError( - "JavaScript application Evidence authority or confidence is invalid", + throw invalidEvidence( + path, + "authority", + "JavaScript application Evidence must use shipped-artifact authority and derived confidence.", ); if (evidence.predicate_type !== "rea.javascript-application-analysis") - throw new TypeError( - "JavaScript application Evidence predicate does not match its result shape", + throw invalidEvidence( + path, + "predicate_type", + "JavaScript application Evidence predicate does not match its result shape.", ); - analyzeJavaScriptApplicationInputSchema.parse({ - input_path: result.input_path, - ...evidence.parameters, - }); - assertApplicationSubject(evidence, result); + parseCallerValue( + (input) => analyzeJavaScriptApplicationInputSchema.parse(input), + { input_path: result.input_path, ...evidence.parameters }, + [...path, "parameters"], + ); + assertApplicationSubject(evidence, result, path); return { evidence, graph: result.graph, @@ -126,30 +168,40 @@ const staticApplicationSourceForResult = ( const staticRuntimeSource = ( evidence: Evidence, + path: readonly (string | number)[], ): ApplicationGraphEvidenceSource => { if ( evidence.authority !== "analyst-inference" || evidence.confidence !== "inferred" ) - throw new TypeError( - "JavaScript runtime reconciliation Evidence authority or confidence is invalid", + throw invalidEvidence( + path, + "authority", + "Runtime reconciliation Evidence must use analyst-inference authority and inferred confidence.", ); - const result = javascriptRuntimeReconciliationResultSchema.parse( + const result = parseCallerValue( + (input) => javascriptRuntimeReconciliationResultSchema.parse(input), evidence.normalized_result, + [...path, "normalized_result"], ); - if ( - result.evidence_links.some( - (evidenceId) => !evidence.evidence_links.includes(evidenceId), - ) - ) - throw new TypeError( - "Runtime reconciliation result references Evidence outside its envelope", + const unmatchedLink = result.evidence_links.find( + (evidenceId) => !evidence.evidence_links.includes(evidenceId), + ); + if (unmatchedLink !== undefined) + throw invalidEvidence( + [...path, "normalized_result"], + "evidence_links", + `Runtime reconciliation result references Evidence ${unmatchedLink} outside its envelope.`, ); const applicationLayer = result.static_layers.find( ({ role }) => role === "application", ); if (applicationLayer === undefined) - throw new TypeError("Runtime reconciliation application layer is missing"); + throw invalidEvidence( + [...path, "normalized_result"], + "static_layers", + "Runtime reconciliation Evidence must include its application layer.", + ); return { evidence, graph: result.graph, @@ -161,24 +213,30 @@ const staticRuntimeSource = ( const managedApplicationSource = ( evidence: Evidence, + path: readonly (string | number)[], ): ApplicationGraphEvidenceSource => { if ( evidence.authority !== "analyst-inference" || evidence.confidence !== "inferred" ) - throw new TypeError( - "Managed application graph Evidence authority or confidence is invalid", + throw invalidEvidence( + path, + "authority", + "Managed application graph Evidence must use analyst-inference authority and inferred confidence.", ); - const result = managedApplicationGraphResultSchema.parse( + const result = parseCallerValue( + (input) => managedApplicationGraphResultSchema.parse(input), evidence.normalized_result, + [...path, "normalized_result"], ); - if ( - result.evidence_links.some( - (evidenceId) => !evidence.evidence_links.includes(evidenceId), - ) - ) - throw new TypeError( - "Managed application graph result references Evidence outside its envelope", + const unmatchedLink = result.evidence_links.find( + (evidenceId) => !evidence.evidence_links.includes(evidenceId), + ); + if (unmatchedLink !== undefined) + throw invalidEvidence( + [...path, "normalized_result"], + "evidence_links", + `Managed application graph result references Evidence ${unmatchedLink} outside its envelope.`, ); return { evidence, @@ -192,6 +250,7 @@ const managedApplicationSource = ( const assertApplicationSubject = ( evidence: Evidence, result: JavaScriptApplicationAnalysisResult, + path: readonly (string | number)[], ): void => { if ( evidence.subject === null || @@ -199,8 +258,10 @@ const assertApplicationSubject = ( evidence.subject.format !== result.format || evidence.subject.local_path !== result.input_path ) - throw new TypeError( - "JavaScript application Evidence subject does not match its result", + throw invalidEvidence( + path, + "subject", + "JavaScript application Evidence subject must match the analyzed artifact identity and input path.", ); }; @@ -216,15 +277,116 @@ const providerMatches = ( evidence.provider.name === provider.name && evidence.provider.version === provider.version; +const requirePredicate = ( + evidence: Evidence, + expected: string, + path: readonly (string | number)[], +): void => { + if (evidence.predicate_type !== expected) + throw invalidEvidence( + path, + "predicate_type", + `Expected predicate ${expected} for operation ${evidence.operation}.`, + ); +}; + +const requireProvider = ( + evidence: Evidence, + expected: { + readonly id: string; + readonly name: string; + readonly version: string; + }, + path: readonly (string | number)[], +): void => { + if (!providerMatches(evidence, expected)) + throw invalidEvidence( + path, + "provider", + `Expected provider ${expected.id} v${expected.version} for operation ${evidence.operation}.`, + ); +}; + /** Parse unique, artifact-bound Evidence that may extend a native handoff. */ export const parseNativeApplicationEvidence = ( inputs: readonly unknown[], -): Evidence[] => { - const parsed = inputs.map((input) => parseEvidence(input)); - if (parsed.some(({ subject }) => subject === null)) - throw new TypeError("Native handoff Evidence requires an artifact subject"); - const ids = parsed.map(({ evidence_id: id }) => id); - if (new Set(ids).size !== ids.length) - throw new TypeError("Native handoff Evidence must be unique"); - return parsed; + path: readonly (string | number)[] = [], +): Result => + captureInputValidation(() => { + const parsed = inputs.map((input, index) => + parseEvidenceInput(input, [...path, index]), + ); + const withoutSubject = parsed.findIndex(({ subject }) => subject === null); + if (withoutSubject !== -1) + throw invalidEvidence( + [...path, withoutSubject], + "subject", + "Native handoff Evidence requires an artifact subject.", + ); + const ids = parsed.map(({ evidence_id: id }) => id); + if (new Set(ids).size !== ids.length) + throw invalidEvidence( + path, + "evidence_id", + "Native handoff Evidence must be unique.", + ); + return parsed; + }); + +const captureInputValidation = ( + operation: () => Value, +): Result => { + try { + return ok(operation()); + } catch (cause: unknown) { + if (cause instanceof InvalidApplicationGraphEvidence) + return err({ issues: cause.issues }); + throw cause; + } }; + +const parseEvidenceInput = ( + input: unknown, + path: readonly (string | number)[], +): Evidence => { + try { + return parseEvidence(input); + } catch (cause: unknown) { + if (cause instanceof z.ZodError) throw zodInputFailure(cause, input, path); + throw cause; + } +}; + +const parseCallerValue = ( + parse: (input: unknown) => Value, + input: unknown, + path: readonly (string | number)[], +): Value => { + try { + return parse(input); + } catch (cause: unknown) { + if (cause instanceof z.ZodError) throw zodInputFailure(cause, input, path); + throw cause; + } +}; + +const zodInputFailure = ( + cause: z.ZodError, + input: unknown, + path: readonly (string | number)[], +): InvalidApplicationGraphEvidence => + new InvalidApplicationGraphEvidence( + projectInputIssues(cause.issues, input).map((issue) => ({ + ...issue, + path: [...path, ...issue.path], + })), + ); + +const invalidEvidence = ( + path: readonly (string | number)[], + field: string, + message: string, +): InvalidApplicationGraphEvidence => + new InvalidApplicationGraphEvidence([ + { path: [...path, field], reason: "invalid_value", message }, + ]); diff --git a/src/application/javascript/JavaScriptApplicationWorkflowService.ts b/src/application/javascript/JavaScriptApplicationWorkflowService.ts index a9c31ed12a..f85330cee2 100644 --- a/src/application/javascript/JavaScriptApplicationWorkflowService.ts +++ b/src/application/javascript/JavaScriptApplicationWorkflowService.ts @@ -17,6 +17,7 @@ import { err, ok, type Result } from "../../domain/result.js"; import { compareSourceToBundle } from "../../domain/javascript/sourceToBundleComparison.js"; import { compareSourceToBundleInputSchema } from "../../domain/javascript/sourceToBundleComparisonSchemas.js"; import { + applicationGraphEvidenceInputError, parseApplicationGraphEvidence, parseNativeApplicationEvidence, } from "./JavaScriptApplicationEvidenceGraph.js"; @@ -45,11 +46,24 @@ export const traceApplicationFeatureEvidenceValidated = ( input: z.output, ): Result => { const operation = "trace_application_feature"; - try { - const source = parseApplicationGraphEvidence(input.application); - const nativeEvidence = parseNativeApplicationEvidence( - input.native_observations, + const sourceResult = parseApplicationGraphEvidence(input.application, [ + "application", + ]); + if (!sourceResult.ok) + return err( + applicationGraphEvidenceInputError(operation, sourceResult.error), ); + const nativeResult = parseNativeApplicationEvidence( + input.native_observations, + ["native_observations"], + ); + if (!nativeResult.ok) + return err( + applicationGraphEvidenceInputError(operation, nativeResult.error), + ); + try { + const source = sourceResult.value; + const nativeEvidence = nativeResult.value; const result = traceApplicationFeature({ sourceEvidenceId: source.evidence.evidence_id, graph: source.graph, @@ -69,7 +83,7 @@ export const traceApplicationFeatureEvidenceValidated = ( ), ); } catch (cause: unknown) { - return workflowFailure(operation, cause, input); + return workflowFailure(operation, cause); } }; @@ -91,15 +105,35 @@ export const compareApplicationVersionsEvidenceValidated = ( input: z.output, ): Result => { const operation = "compare_application_versions"; + const leftResult = parseApplicationGraphEvidence(input.left, ["left"]); + if (!leftResult.ok) + return err(applicationGraphEvidenceInputError(operation, leftResult.error)); + const rightResult = parseApplicationGraphEvidence(input.right, ["right"]); + if (!rightResult.ok) + return err( + applicationGraphEvidenceInputError(operation, rightResult.error), + ); + const leftNativeResult = parseNativeApplicationEvidence( + input.left_native_observations, + ["left_native_observations"], + ); + if (!leftNativeResult.ok) + return err( + applicationGraphEvidenceInputError(operation, leftNativeResult.error), + ); + const rightNativeResult = parseNativeApplicationEvidence( + input.right_native_observations, + ["right_native_observations"], + ); + if (!rightNativeResult.ok) + return err( + applicationGraphEvidenceInputError(operation, rightNativeResult.error), + ); try { - const left = parseApplicationGraphEvidence(input.left); - const right = parseApplicationGraphEvidence(input.right); - const leftNative = parseNativeApplicationEvidence( - input.left_native_observations, - ); - const rightNative = parseNativeApplicationEvidence( - input.right_native_observations, - ); + const left = leftResult.value; + const right = rightResult.value; + const leftNative = leftNativeResult.value; + const rightNative = rightNativeResult.value; const result = compareJavaScriptApplicationVersions({ left: { evidenceId: left.evidence.evidence_id, @@ -128,7 +162,7 @@ export const compareApplicationVersionsEvidenceValidated = ( ), ); } catch (cause: unknown) { - return workflowFailure(operation, cause, input); + return workflowFailure(operation, cause); } }; @@ -137,8 +171,15 @@ export const compareSourceToBundleEvidenceValidated = ( input: z.output, ): Result => { const operation = "compare_source_to_bundle"; + const applicationResult = parseApplicationGraphEvidence(input.application, [ + "application", + ]); + if (!applicationResult.ok) + return err( + applicationGraphEvidenceInputError(operation, applicationResult.error), + ); try { - const application = parseApplicationGraphEvidence(input.application); + const application = applicationResult.value; const result = compareSourceToBundle({ reference: input.reference, application: { @@ -157,7 +198,7 @@ export const compareSourceToBundleEvidenceValidated = ( ), ); } catch (cause: unknown) { - return workflowFailure(operation, cause, input); + return workflowFailure(operation, cause); } }; @@ -182,9 +223,17 @@ export const compareJavaScriptExportShapesEvidenceValidated = ( input: z.output, ): Result => { const operation = "compare_javascript_export_shapes"; + const leftResult = parseApplicationGraphEvidence(input.left, ["left"]); + if (!leftResult.ok) + return err(applicationGraphEvidenceInputError(operation, leftResult.error)); + const rightResult = parseApplicationGraphEvidence(input.right, ["right"]); + if (!rightResult.ok) + return err( + applicationGraphEvidenceInputError(operation, rightResult.error), + ); try { - const left = parseApplicationGraphEvidence(input.left); - const right = parseApplicationGraphEvidence(input.right); + const left = leftResult.value; + const right = rightResult.value; const result = compareJavaScriptExportShapes({ left: { evidenceId: left.evidence.evidence_id, @@ -213,50 +262,17 @@ export const compareJavaScriptExportShapesEvidenceValidated = ( ), ); } catch (cause: unknown) { - return workflowFailure(operation, cause, input); + return workflowFailure(operation, cause); } }; const workflowFailure = ( operation: string, cause: unknown, - input: unknown, ): Result => - cause instanceof z.ZodError - ? err(analysisInputErrorFromIssues(operation, cause.issues, input)) - : cause instanceof TypeError && - SAFE_APPLICATION_INPUT_CONSTRAINTS.has(cause.message) - ? err( - new AnalysisInputError(operation, undefined, [ - { path: [], reason: "invalid_value", message: cause.message }, - ]), - ) - : err( - new AnalysisProtocolError( - "JavaScript application workflow produced an invalid result", - { cause }, - ), - ); - -const SAFE_APPLICATION_INPUT_CONSTRAINTS = new Set([ - "Evidence semantic identifier does not match its record", - "Application workflow requires authenticated analyze_javascript_application, reconcile_javascript_runtime, or project_managed_application_graph Evidence", - "JavaScript application Evidence authority or confidence is invalid", - "JavaScript application Evidence predicate does not match its result shape", - "JavaScript application Evidence subject does not match its result", - "Runtime reconciliation Evidence authority or confidence is invalid", - "Runtime reconciliation result references Evidence outside its envelope", - "Runtime reconciliation application layer is missing", - "Managed application graph Evidence authority or confidence is invalid", - "Managed application graph result references Evidence outside its envelope", - "Native handoff Evidence requires an artifact subject", - "Native handoff Evidence must be unique", - ...[ - "analyze_javascript_application", - "reconcile_javascript_runtime", - "project_managed_application_graph", - ].map( - (operation) => - `Evidence does not match the supported ${operation} contract`, - ), -]); + err( + new AnalysisProtocolError( + `JavaScript application workflow failed while producing ${operation}`, + { cause }, + ), + ); diff --git a/src/application/javascript/JavaScriptRuntimeReconciliationService.ts b/src/application/javascript/JavaScriptRuntimeReconciliationService.ts index bc44855521..d392ef1821 100644 --- a/src/application/javascript/JavaScriptRuntimeReconciliationService.ts +++ b/src/application/javascript/JavaScriptRuntimeReconciliationService.ts @@ -85,7 +85,6 @@ const isPortableAbsolutePath = (path: string): boolean => /^(?:\\\\|\/\/)[^\\/]+[\\/][^\\/]+(?:[\\/]|$)/u.test(path); const SAFE_RECONCILIATION_INPUT_CONSTRAINTS = new Set([ - "Evidence semantic identifier does not match its record", "JavaScript application Evidence subject disagrees with its result", "Runtime reconciliation requires inspect_web_page, inspect_electron_page, observe_javascript_runtime, or capture_electron_scenario Evidence", "Runtime Evidence target or declared role disagrees with its result", diff --git a/src/application/javascript/JavaScriptSemanticTraceService.ts b/src/application/javascript/JavaScriptSemanticTraceService.ts index 98dfb12c08..24413df439 100644 --- a/src/application/javascript/JavaScriptSemanticTraceService.ts +++ b/src/application/javascript/JavaScriptSemanticTraceService.ts @@ -5,6 +5,10 @@ import { AnalysisProtocolError, } from "../../domain/analysisErrorCore.js"; import { type AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + applicationGraphEvidenceInputError, + parseApplicationGraphEvidence, +} from "./JavaScriptApplicationEvidenceGraph.js"; import type { Evidence } from "../../domain/evidence.js"; import { jsonValueSchema } from "../../domain/jsonValue.js"; import { queryJavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticQuery.js"; @@ -13,7 +17,6 @@ import { traceJavaScriptSemanticsInputSchema, } from "../../domain/javascript/javascriptSemanticTraceSchemas.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { parseApplicationGraphEvidence } from "./JavaScriptApplicationEvidenceGraph.js"; import { createJavaScriptSemanticTraceEvidence } from "./JavaScriptApplicationWorkflowEvidence.js"; const OPERATION = "trace_javascript_semantics" as const; @@ -22,19 +25,26 @@ const OPERATION = "trace_javascript_semantics" as const; export const traceJavaScriptSemanticsEvidenceValidated = ( input: z.output, ): Result => { + const sourceResult = parseApplicationGraphEvidence(input.application, [ + "application", + ]); + if (!sourceResult.ok) + return err( + applicationGraphEvidenceInputError(OPERATION, sourceResult.error), + ); + const source = sourceResult.value; + if (source.semanticGraph === null) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["application"], + reason: "invalid_value", + expected: + "inline Evidence from analyze_javascript_application with a semantic graph", + }, + ]), + ); try { - const source = parseApplicationGraphEvidence(input.application); - if (source.semanticGraph === null) - return err( - new AnalysisInputError(OPERATION, undefined, [ - { - path: ["application"], - reason: "invalid_value", - expected: - "inline Evidence from analyze_javascript_application with a semantic graph", - }, - ]), - ); const query = queryJavaScriptSemanticGraph( source.semanticGraph, input.query, diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 70d3e8c90b..fe6c692a8c 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -140,6 +140,18 @@ export const evidenceSchema = evidenceBaseSchema.superRefine( validateAnalysisProfileProvider, ); +const authenticatedEvidenceSchema = evidenceSchema.superRefine( + (evidence, context) => { + const { evidence_id: evidenceId, ...withoutId } = evidence; + if (computeEvidenceId(withoutId) !== evidenceId) + context.addIssue({ + code: "custom", + path: ["evidence_id"], + message: "Evidence semantic identifier does not match its record", + }); + }, +); + /** Complete observation whose normalized payload retains its operation-specific type. */ export type Evidence = z.infer< typeof evidenceSchema @@ -224,13 +236,7 @@ export const parseEvidence = (input: unknown): Evidence => { ? immutableEvidenceSnapshots.get(input) : undefined; if (immutable !== undefined) return immutable; - const evidence = evidenceSchema.parse(input); - const { evidence_id: evidenceId, ...withoutId } = evidence; - if (computeEvidenceId(withoutId) !== evidenceId) - throw new TypeError( - "Evidence semantic identifier does not match its record", - ); - return evidence; + return authenticatedEvidenceSchema.parse(input); }; /** Authenticate and seal a ledger-owned snapshot; external mutable values are copied first. */ diff --git a/src/domain/evidenceBundle.ts b/src/domain/evidenceBundle.ts index c914ab3199..39dcf43e7d 100644 --- a/src/domain/evidenceBundle.ts +++ b/src/domain/evidenceBundle.ts @@ -243,7 +243,20 @@ export const parseEvidenceBundle = (input: unknown): EvidenceBundle => { throw new TypeError("Evidence bundle contains duplicate record IDs"); validateUnknownGraph(parsed.unknowns, parsed.records); const canonical = createEvidenceBundle( - parsed.records.map(parseEvidence), + parsed.records.map((record, index) => { + try { + return parseEvidence(record); + } catch (cause: unknown) { + if (cause instanceof z.ZodError) + throw new z.ZodError( + cause.issues.map((issue) => ({ + ...issue, + path: ["records", index, ...issue.path], + })), + ); + throw cause; + } + }), parsed.unknowns, ); if (JSON.stringify(parsed) !== JSON.stringify(canonical)) diff --git a/tests/boundary/filesystem/evidenceBundleFiles.test.ts b/tests/boundary/filesystem/evidenceBundleFiles.test.ts index 11216b6718..d5a92938af 100644 --- a/tests/boundary/filesystem/evidenceBundleFiles.test.ts +++ b/tests/boundary/filesystem/evidenceBundleFiles.test.ts @@ -313,7 +313,7 @@ describe("evidence bundle filesystem adapter", () => { error: { _tag: "EvidenceIntegrityError", userMessage: expect.stringContaining( - "Evidence semantic identifier does not match its record", + "at records.0.evidence_id: Evidence semantic identifier does not match its record", ), }, }); diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index 5276b510a2..8da2a46a53 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -320,7 +320,7 @@ describe("complete comparison projections", () => { }); describe("JavaScript application workflow error diagnostics", () => { - it("returns safe explicit Evidence constraints from graph workflows", async () => { + it("identifies the inconsistent Evidence subject in graph workflows", async () => { const root = await createTestTempDirectory( "rea-application-workflow-diagnostics-", ); @@ -328,10 +328,10 @@ describe("JavaScript application workflow error diagnostics", () => { const evidence = await analyzeFixture(root); const subject = evidence.subject; if (subject === null) throw new Error("Analysis Evidence subject missing"); - const secretPath = "/private/token=do-not-return"; + const inconsistentPath = "/different/application.js"; const inconsistent = { ...evidence, - subject: { ...subject, local_path: secretPath }, + subject: { ...subject, local_path: inconsistentPath }, }; const result = traceApplicationFeatureEvidence({ @@ -346,11 +346,10 @@ describe("JavaScript application workflow error diagnostics", () => { expect(result.error.issues).toContainEqual( expect.objectContaining({ reason: "invalid_value", - message: - "JavaScript application Evidence subject does not match its result", + path: ["application", "subject"], + message: expect.stringContaining("subject must match"), }), ); - expect(JSON.stringify(result.error.issues)).not.toContain(secretPath); }); it("preserves schema-authored Evidence validation messages", async () => { diff --git a/tests/boundary/filesystem/javascriptExportContainerCoverage.test.ts b/tests/boundary/filesystem/javascriptExportContainerCoverage.test.ts index f410e4f29d..bea6adbac1 100644 --- a/tests/boundary/filesystem/javascriptExportContainerCoverage.test.ts +++ b/tests/boundary/filesystem/javascriptExportContainerCoverage.test.ts @@ -232,7 +232,9 @@ const analyzeReturn = async (root: string, properties: string) => { await writeFixtureFiles(root, { "parser.mjs": `${prefix}${expression});\n` }); const analyzed = await analyzeJavaScriptApplication({ input_path: root }); if (!analyzed.ok) throw analyzed.error; - const source = parseApplicationGraphEvidence(analyzed.value); + const parsed = parseApplicationGraphEvidence(analyzed.value); + if (!parsed.ok) throw new Error("Analysis Evidence must parse"); + const source = parsed.value; const observation = source.graph.nodes .flatMap(({ observations }) => observations) .find( diff --git a/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts b/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts index 5be94b62d3..0039be5c1a 100644 --- a/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts +++ b/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts @@ -938,7 +938,9 @@ const analyzeGraph = async (root: string) => { input_path: root, }); if (!result.ok) throw result.error; - return parseApplicationGraphEvidence(result.value); + const parsed = parseApplicationGraphEvidence(result.value); + if (!parsed.ok) throw new Error("Analysis Evidence must parse"); + return parsed.value; }; const analyzeSources = async ( diff --git a/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts b/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts index eccc05883f..77d383cbef 100644 --- a/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts +++ b/tests/boundary/filesystem/javascriptRequireLiteralPaths.test.ts @@ -15,7 +15,9 @@ const analyze = async (root: string) => { format: "directory", }); if (!result.ok) throw result.error; - const authenticated = parseApplicationGraphEvidence(result.value); + const parsed = parseApplicationGraphEvidence(result.value); + if (!parsed.ok) throw new Error("Analysis Evidence must parse"); + const authenticated = parsed.value; return { graph: authenticated.graph, result: javascriptApplicationAnalysisResultSchema.parse( diff --git a/tests/boundary/filesystem/sourceToBundlePaths.test.ts b/tests/boundary/filesystem/sourceToBundlePaths.test.ts index 3514e5ead1..7dcd910ffe 100644 --- a/tests/boundary/filesystem/sourceToBundlePaths.test.ts +++ b/tests/boundary/filesystem/sourceToBundlePaths.test.ts @@ -668,9 +668,16 @@ it("rejects static application Evidence whose subject path disagrees with its re }, }; - expect(() => parseApplicationGraphEvidence(inconsistent)).toThrow( - "JavaScript application Evidence subject does not match its result", - ); + const parsed = parseApplicationGraphEvidence(inconsistent); + expect(parsed.ok).toBe(false); + if (!parsed.ok) + expect(parsed.error.issues).toContainEqual( + expect.objectContaining({ + path: ["subject"], + reason: "invalid_value", + message: expect.stringContaining("subject must match"), + }), + ); }); const compareTrees = async (previous: string, current: string) => { @@ -682,9 +689,11 @@ const compareTrees = async (previous: string, current: string) => { if (!reference.ok) throw new Error(reference.error.message); const analyzed = await analyzeJavaScriptApplication({ input_path: current }); if (!analyzed.ok) throw analyzed.error; - const application = parseApplicationGraphEvidence( + const parsedApplication = parseApplicationGraphEvidence( JSON.parse(JSON.stringify(analyzed.value)), ); + if (!parsedApplication.ok) throw new Error("Analysis Evidence must parse"); + const application = parsedApplication.value; const comparison = compareSourceToBundle({ reference: reference.value, application: { diff --git a/tests/boundary/javascriptSemanticTraceEvidence.test.ts b/tests/boundary/javascriptSemanticTraceEvidence.test.ts new file mode 100644 index 0000000000..def3bde04d --- /dev/null +++ b/tests/boundary/javascriptSemanticTraceEvidence.test.ts @@ -0,0 +1,221 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; +import { Ajv2020 } from "ajv/dist/2020.js"; + +import { createApplicationMcpHarness } from "../fixtures/applicationMcpHarness.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { createCli } from "../../src/cli.js"; +import { createEvidence, parseEvidence } from "../../src/domain/evidence.js"; +import type { JsonValue } from "../../src/domain/jsonValue.js"; +import { analysisCliErrorEnvelopeSchema } from "../../src/contracts/errorSchemas.js"; +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { javaScriptSemanticTraceResultSchema } from "../../src/domain/javascript/javascriptSemanticTraceSchemas.js"; +import { resolveJavaScriptSemanticEvidence } from "../../src/domain/javascript/javascriptSemanticGraph.js"; + +const SEMANTIC_GRAPH_ID = `jsrg_${"0".repeat(64)}`; + +it("reports invalid nested semantic graph Evidence as caller input through CLI and SDK MCP", async () => { + const root = await createTestTempDirectory("rea-semantic-trace-input-"); + await mkdir(join(root, "src")); + await writeFile( + join(root, "src", "main.js"), + "export function parse(value) { return JSON.parse(value); }\n", + ); + + const cli = createCli({}); + const analyzed = await serveCli(cli, [ + "analyze-javascript-application", + root, + "--json", + ]); + expect(analyzed.exitCode).toBe(0); + const sourceEvidence = parseEvidence(JSON.parse(analyzed.stdout)); + const normalizedResult = structuredClone(sourceEvidence.normalized_result); + if (!isJsonObject(normalizedResult)) + throw new Error("CLI analysis result must be an object"); + const semanticGraph = normalizedResult.semantic_graph; + if (!isJsonObject(semanticGraph)) + throw new Error("CLI analysis must include a semantic graph"); + const firstNode = semanticGraph.nodes; + if (!Array.isArray(firstNode) || !isJsonObject(firstNode[0])) + throw new Error("CLI semantic graph must include a node"); + const seedNodeId = firstNode[0].node_id; + if (typeof seedNodeId !== "string") + throw new Error("CLI semantic graph node must have an identifier"); + semanticGraph.graph_id = SEMANTIC_GRAPH_ID; + + const subject = sourceEvidence.subject; + if (subject === null) throw new Error("CLI Evidence must include a subject"); + const malformedEvidence = createEvidence( + { + path: subject.local_path, + sha256: subject.digest.sha256, + format: subject.format, + ...(subject.architecture === null + ? {} + : { architecture: subject.architecture }), + }, + sourceEvidence.provider, + { + predicateType: sourceEvidence.predicate_type, + operation: sourceEvidence.operation, + parameters: sourceEvidence.parameters, + result: normalizedResult, + confidence: sourceEvidence.confidence, + authority: sourceEvidence.authority, + }, + ); + expect(malformedEvidence.evidence_id).not.toBe(sourceEvidence.evidence_id); + + const arguments_ = { + application: malformedEvidence, + query: { + seed: { kind: "semantic-node", node_id: seedNodeId }, + direction: "backward-provenance", + }, + }; + const cliTrace = await serveCli(cli, [ + "trace-javascript-semantics", + JSON.stringify(arguments_), + "--json", + ]); + const cliEnvelope = analysisCliErrorEnvelopeSchema.parse( + JSON.parse(cliTrace.stdout), + ); + const { error: _label, ...cliError } = cliEnvelope; + expect(cliError).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + details: { + issues: [ + { + path: [ + "application", + "normalized_result", + "semantic_graph", + "graph_id", + ], + reason: "invalid_value", + }, + ], + }, + }); + + const harness = await createApplicationMcpHarness(); + try { + await assertSuccessfulTraceParity( + cli, + harness, + sourceEvidence, + arguments_.query, + ); + + const mcpTrace = await harness.client.callTool({ + name: "trace_javascript_semantics", + arguments: arguments_, + }); + const { error: mcpError } = parseMcpToolError(mcpTrace); + expect(mcpError).toEqual(cliError); + } finally { + await harness.close(); + } +}); + +const assertSuccessfulTraceParity = async ( + cli: ReturnType, + harness: Awaited>, + application: ReturnType, + query: { readonly seed: unknown; readonly direction: string }, +): Promise => { + const arguments_ = { application, query }; + const cliResponse = await serveCli(cli, [ + "trace-javascript-semantics", + JSON.stringify(arguments_), + "--json", + ]); + const cliEvidence = parseEvidence(JSON.parse(cliResponse.stdout)); + const cliResult = javaScriptSemanticTraceResultSchema.parse( + cliEvidence.normalized_result, + ); + assertTraceContextsResolve(cliResult); + + const mcpResponse = await harness.client.callTool({ + name: "trace_javascript_semantics", + arguments: arguments_, + }); + expect(mcpResponse.isError).not.toBe(true); + const { tools } = await harness.client.listTools(); + const advertised = tools.find( + ({ name }) => name === "trace_javascript_semantics", + ); + if (advertised?.outputSchema === undefined) + throw new Error("Semantic trace SDK output schema is missing"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(outputSchema)).toBe(true); + expect(ajv.validate(outputSchema, mcpResponse.structuredContent)).toBe(true); + const mcpEvidence = toolContract( + "trace_javascript_semantics", + ).outputSchema.parse(mcpResponse.structuredContent); + expect(mcpEvidence.normalized_result).toEqual(cliResult); + expect(mcpEvidence).toEqual(cliEvidence); +}; + +const assertTraceContextsResolve = ( + result: ReturnType, +): void => { + const contexts = new Map( + result.evidence_contexts.map((context) => [context.context_id, context]), + ); + const references = [ + ...result.nodes.map(({ evidence }) => evidence), + ...result.relations.map(({ evidence }) => evidence), + ...result.unknowns.map(({ evidence }) => evidence), + ]; + const referencedIds = new Set(); + for (const reference of references) { + const context = contexts.get(reference.context_id); + expect(context, reference.context_id).toBeDefined(); + if (context === undefined) continue; + referencedIds.add(reference.context_id); + expect( + resolveJavaScriptSemanticEvidence( + { evidence_contexts: result.evidence_contexts }, + reference, + ), + ).toMatchObject({ + ...Object.fromEntries( + Object.entries(context).filter(([key]) => key !== "context_id"), + ), + location: reference.location, + }); + } + expect([...referencedIds].toSorted()).toEqual( + [...contexts.keys()].toSorted(), + ); + expect(contexts.size).toBeGreaterThan(0); +}; + +const serveCli = async ( + cli: ReturnType, + arguments_: readonly string[], +): Promise<{ readonly stdout: string; readonly exitCode: number }> => { + let stdout = ""; + let exitCode = 0; + await cli.serve([...arguments_], { + env: {}, + exit: (code) => { + exitCode = code; + }, + stdout: (text) => { + stdout += text; + }, + }); + return { stdout, exitCode }; +}; + +const isJsonObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts index 7667b1ffb1..f1aadf9738 100644 --- a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts +++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts @@ -175,7 +175,7 @@ it("names the side and constraint when Evidence is not a usable capture", async error: { code: "evidence_integrity_mismatch", message: expect.stringContaining( - "The right Evidence failed validation (Evidence semantic identifier does not match its record)", + "The right Evidence failed validation (at evidence_id: Evidence semantic identifier does not match its record)", ), }, });