From e1094313f504a2ed3611b3c18d31120bc4fe847b Mon Sep 17 00:00:00 2001 From: Salman Almutairi <141588124+salmanalmutairi1@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:56:58 +0300 Subject: [PATCH] fix(changed-behavior): withhold equality for unknowns and contradictions (#7624) find_changed_behavior derived behavior_status from each process comparison's aggregate status alone. An unchanged comparison whose first_divergence was unknown (residual unknowns prevent proving no divergence) and a consistent contradiction comparison both fell through to observed_unchanged, with no unresolved finding and no changed-behavior registry unknown. An unchanged comparison with unknown divergence now counts as unknown and adds one comparison-level unresolved first_divergence finding carrying its reason, as artifact comparisons already do for coverage. A contradiction status makes the aggregate unknown, and contradiction dimensions are classified and counted as contradictions instead of unresolved branches. Complete unchanged comparisons stay observed_unchanged and changed comparisons with unknown localization stay observed_changed. Fixes #7384 Fixes #7383 Co-authored-by: Claude Opus 5.5 --- src/domain/changedBehavior.test.ts | 89 +++++++++++++++++++++++++++--- src/domain/changedBehavior.ts | 40 ++++++++++++-- 2 files changed, 116 insertions(+), 13 deletions(-) diff --git a/src/domain/changedBehavior.test.ts b/src/domain/changedBehavior.test.ts index 049f84da9a..84d8e94c72 100644 --- a/src/domain/changedBehavior.test.ts +++ b/src/domain/changedBehavior.test.ts @@ -69,13 +69,21 @@ const comparison = ( ); }; +type FixtureStatus = + | "unchanged" + | "changed" + | "unknown" + | "truncated" + | "contradiction"; + const processResult = ( overrides: Partial<{ - status: "unchanged" | "changed" | "unknown" | "truncated"; - terminal: "unchanged" | "changed" | "unknown" | "truncated"; - interaction: "unchanged" | "changed" | "unknown" | "truncated"; - exit: "unchanged" | "changed" | "unknown" | "truncated"; - process: "unchanged" | "changed" | "unknown" | "truncated"; + status: FixtureStatus; + terminal: FixtureStatus; + interaction: FixtureStatus; + exit: FixtureStatus; + process: FixtureStatus; + first_divergence: JsonValue; }> = {}, ): JsonValue => ({ status: overrides.status ?? "unchanged", @@ -85,7 +93,8 @@ const processResult = ( filesystem: "unchanged", process: overrides.process ?? "unchanged", first_divergence: - overrides.status === "changed" + overrides.first_divergence ?? + (overrides.status === "changed" ? { status: "found", dimension: "terminal", @@ -95,9 +104,11 @@ const processResult = ( left: "left", right: "right", } - : overrides.status === "truncated" || overrides.status === "unknown" + : overrides.status === "truncated" || + overrides.status === "unknown" || + overrides.status === "contradiction" ? { status: "unknown", reason: "Incomplete fixture." } - : { status: "none" }, + : { status: "none" }), limitations: [], }); @@ -230,6 +241,68 @@ describe("changed behavior", () => { }); }); +describe("changed behavior withheld equality", () => { + it("withholds equality when residual unknowns prevent proving no divergence", () => { + const reason = + "Residual unknowns prevent proving that no divergence occurred."; + const evidence = comparison( + "compare_process_captures", + processResult({ first_divergence: { status: "unknown", reason } }), + ); + expect(aggregate([evidence]).result).toMatchObject({ + behavior_status: "unknown", + summary: { observed_changes: 0, contradictions: 0, unresolved: 1 }, + findings: { + items: [ + { + scope: "runtime", + dimension: "first_divergence", + classification: "unresolved_branch", + status: "unknown", + source_comparison_id: evidence.evidence_id, + limitations: [reason], + }, + ], + }, + }); + }); + + it("keeps a changed comparison observed when only its localization is unknown", () => { + const evidence = comparison( + "compare_process_captures", + processResult({ + status: "changed", + terminal: "changed", + first_divergence: { status: "unknown", reason: "Unlocalized." }, + }), + ); + expect(aggregate([evidence]).result).toMatchObject({ + behavior_status: "observed_changed", + summary: { observed_changes: 1, unresolved: 0 }, + }); + }); + + it("reports a process contradiction as a contradiction, never equality", () => { + const evidence = comparison( + "compare_process_captures", + processResult({ status: "contradiction", terminal: "contradiction" }), + ); + expect(aggregate([evidence]).result).toMatchObject({ + behavior_status: "unknown", + summary: { observed_changes: 0, contradictions: 1, unresolved: 0 }, + findings: { + items: [ + { + dimension: "terminal", + classification: "contradiction", + status: "contradiction", + }, + ], + }, + }); + }); +}); + describe("changed-behavior input and Evidence validation", () => { it("rejects an empty comparison selection", () => { expect(() => findChangedBehavior({ comparisons: [] })).toThrowError( diff --git a/src/domain/changedBehavior.ts b/src/domain/changedBehavior.ts index ef3c03180a..86ae8b40ea 100644 --- a/src/domain/changedBehavior.ts +++ b/src/domain/changedBehavior.ts @@ -115,7 +115,8 @@ export const changedBehaviorResultSchema = z export type ChangedBehaviorResult = z.infer; type Finding = z.infer; -type RuntimeStatus = z.infer["status"]; +type ProcessComparison = z.infer; +type RuntimeStatus = ProcessComparison["status"]; /** Find observed runtime changes and separately report static change candidates. */ interface FoundChangedBehavior { @@ -146,7 +147,11 @@ export const findChangedBehavior = (input: unknown): FoundChangedBehavior => { const findings = evidence.flatMap(findingsFor).sort(compareFindings); const runtimeStatuses = evidence.flatMap((comparison) => comparison.operation === "compare_process_captures" - ? [comparison.result.status] + ? [ + unprovenEqualityReason(comparison.result) === undefined + ? comparison.result.status + : "unknown", + ] : [], ); const links = uniqueSorted( @@ -243,7 +248,7 @@ const processFindings = ( ["filesystem", result.filesystem, "resource"], ["process", result.process, "runtime"], ] as const; - return dimensions.flatMap(([dimension, status, scope]) => + const findings = dimensions.flatMap(([dimension, status, scope]) => status === "unchanged" ? [] : [ @@ -254,13 +259,37 @@ const processFindings = ( status, classification: isChange(status) ? "observed_change" - : "unresolved_branch", + : status === "contradiction" + ? "contradiction" + : "unresolved_branch", limitations: result.limitations, }), ], ); + const reason = unprovenEqualityReason(result); + return reason === undefined + ? findings + : [ + ...findings, + makeFinding({ + evidence, + dimension: "first_divergence", + scope: "runtime", + status: "unknown", + classification: "unresolved_branch", + limitations: [...result.limitations, reason], + }), + ]; }; +/** Unchanged dimensions do not establish equality while divergence stays unknown. */ +const unprovenEqualityReason = ( + result: ProcessComparison, +): string | undefined => + result.status === "unchanged" && result.first_divergence.status === "unknown" + ? result.first_divergence.reason + : undefined; + const functionFindings = ( comparison: Extract< ParsedComparisonEvidence, @@ -356,7 +385,8 @@ const behaviorStatus = ( ): ChangedBehaviorResult["behavior_status"] => { if (statuses.length === 0) return "unknown"; if (statuses.includes("truncated")) return "truncated"; - if (statuses.includes("unknown")) return "unknown"; + if (statuses.includes("unknown") || statuses.includes("contradiction")) + return "unknown"; return statuses.some(isChange) ? "observed_changed" : "observed_unchanged"; };