From 554816becc41a91a2a2ee7bbe7c2b1d3abfe4296 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:40:34 +0800 Subject: [PATCH] refactor(verification): centralize native client result discovery --- scripts/verify-claude-client.mjs | 44 ++++----- scripts/verify-copilot-client.mjs | 27 ++---- scripts/verify-deepseek-client.mjs | 29 ++---- scripts/verify-gemini-client.mjs | 53 +++++------ scripts/verify-native-agent-client.mjs | 56 +++++------- scripts/verify-opencode-client.mjs | 27 ++---- scripts/verify-qwen-client.mjs | 45 +++------- scripts/verify/clients/result-discovery.d.mts | 19 ++++ scripts/verify/clients/result-discovery.mjs | 28 ++++++ .../verifiers/clientResultDiscovery.test.ts | 89 +++++++++++++++++++ 10 files changed, 227 insertions(+), 190 deletions(-) create mode 100644 scripts/verify/clients/result-discovery.d.mts create mode 100644 scripts/verify/clients/result-discovery.mjs create mode 100644 tests/conformance/verifiers/clientResultDiscovery.test.ts diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index 4824b60b21..88287d86cc 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -26,6 +26,10 @@ import { expandMcpSessionStatus, sortVerifierNames, } from "./lib/mcp-verifier-results.mjs"; +import { + findClientResult, + isClientJavaScriptEvidence, +} from "./verify/clients/result-discovery.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createAnthropicModelFixture } from "./verify/clients/anthropic-model-fixture.mjs"; @@ -106,28 +110,14 @@ await withVerifierCleanup(async () => { ); return false; }; - const extractInlineResult = (value, matches) => { - if (typeof value === "string") { - try { - // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. - const json = value.replace( - /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, - "", - ); - return extractInlineResult(JSON.parse(json), matches); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if (matches(value)) return value; - for (const nested of Object.values(value)) { - const result = extractInlineResult(nested, matches); - if (result) return result; - } - } - return undefined; - }; + const decodeClaudeResult = (text) => + JSON.parse( + // Native Claude appends this resource hint. Original bodies remain in request artifacts. + text.replace( + /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, + "", + ), + ); fixture = await createAnthropicModelFixture({ directory: lab, onRequest: async ({ body, tools, requestIndex }) => { @@ -181,11 +171,12 @@ await withVerifierCleanup(async () => { } } if (mode === "call" && !sessionStatusSeen) { - const observedStatus = extractInlineResult( + const observedStatus = findClientResult( body.messages, (value) => Boolean(value.result?.server_identity) && Array.isArray(value.result?.tool_availability?.groups), + decodeClaudeResult, ); if (observedStatus) { const parsedStatus = @@ -242,11 +233,10 @@ await withVerifierCleanup(async () => { expectedSkillBody !== undefined && containsText(body.messages, expectedSkillBody) && containsText(body.messages, skillDirectory); - const observed = extractInlineResult( + const observed = findClientResult( body.messages, - (value) => - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0, + isClientJavaScriptEvidence, + decodeClaudeResult, ); if (observed) { const { evidence } = await assertClientJavaScriptFixtureEvidence( diff --git a/scripts/verify-copilot-client.mjs b/scripts/verify-copilot-client.mjs index 71b08fb7a5..acf9f9eba3 100644 --- a/scripts/verify-copilot-client.mjs +++ b/scripts/verify-copilot-client.mjs @@ -19,6 +19,10 @@ import { CLIENT_JAVASCRIPT_FIXTURE_SOURCE, assertClientJavaScriptFixtureEvidence, } from "./verify/clients/javascript-fixture.mjs"; +import { + findClientResult, + isClientJavaScriptEvidence, +} from "./verify/clients/result-discovery.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -124,27 +128,6 @@ await withVerifierCleanup(async () => { ); return false; }; - const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; - }; fixture = await createOpenAiModelFixture({ directory: lab, prefix: "copilot", @@ -204,7 +187,7 @@ await withVerifierCleanup(async () => { expectedSkillBody !== undefined && containsText(body.messages, expectedSkillBody) && containsText(body.messages, skillDirectory); - const observed = extractEvidence(replies); + const observed = findClientResult(replies, isClientJavaScriptEvidence); if (observed) { const { evidence } = await assertClientJavaScriptFixtureEvidence( observed, diff --git a/scripts/verify-deepseek-client.mjs b/scripts/verify-deepseek-client.mjs index 5964eb6301..2a061df833 100644 --- a/scripts/verify-deepseek-client.mjs +++ b/scripts/verify-deepseek-client.mjs @@ -15,6 +15,10 @@ import { CLIENT_JAVASCRIPT_FIXTURE_SOURCE, assertClientJavaScriptFixtureEvidence, } from "./verify/clients/javascript-fixture.mjs"; +import { + findClientResult, + isClientJavaScriptEvidence, +} from "./verify/clients/result-discovery.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -130,26 +134,6 @@ await withVerifierCleanup(async () => { } else validatePatterns(child); } }; - const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes - ) - return value; - for (const child of Object.values(value)) { - const found = extractEvidence(child); - if (found) return found; - } - } - }; fixture = await createOpenAiModelFixture({ directory: lab, prefix: `deepseek-${mode}`, @@ -192,7 +176,10 @@ await withVerifierCleanup(async () => { '', ) && serialized.includes(dirname(skillPath)); for (const message of toolMessages) { - const found = extractEvidence(message.content); + const found = findClientResult( + message.content, + isClientJavaScriptEvidence, + ); if (found === undefined) continue; const { evidence } = await assertClientJavaScriptFixtureEvidence( found, diff --git a/scripts/verify-gemini-client.mjs b/scripts/verify-gemini-client.mjs index 9e99a69a62..8ed5d94fab 100644 --- a/scripts/verify-gemini-client.mjs +++ b/scripts/verify-gemini-client.mjs @@ -21,6 +21,10 @@ import { CLIENT_JAVASCRIPT_FIXTURE_SOURCE, assertClientJavaScriptFixtureEvidence, } from "./verify/clients/javascript-fixture.mjs"; +import { + findClientResult, + isClientJavaScriptEvidence, +} from "./verify/clients/result-discovery.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs"; @@ -231,37 +235,18 @@ await withVerifierCleanup(async () => { } return undefined; }; - const extractEvidence = (value) => { - if (typeof value === "string") { - if ( - value.startsWith("\n") && - value.endsWith("\n") - ) { - return extractEvidence( - value.slice( - "\n".length, - -"\n".length, - ), - ); - } - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; + const decodeGeminiResult = (text) => { + if ( + text.startsWith("\n") && + text.endsWith("\n") + ) + return decodeGeminiResult( + text.slice( + "\n".length, + -"\n".length, + ), + ); + return JSON.parse(text); }; fixture = await createGeminiModelFixture({ directory: lab, @@ -327,7 +312,11 @@ await withVerifierCleanup(async () => { join(profile, ".agents", "skills", "reverse-engineer-anything"), ), ); - const observed = extractEvidence(responses); + const observed = findClientResult( + responses, + isClientJavaScriptEvidence, + decodeGeminiResult, + ); if (observed) { const { evidence } = await assertClientJavaScriptFixtureEvidence( observed, diff --git a/scripts/verify-native-agent-client.mjs b/scripts/verify-native-agent-client.mjs index e4e62b3756..bed1f8b41f 100644 --- a/scripts/verify-native-agent-client.mjs +++ b/scripts/verify-native-agent-client.mjs @@ -12,6 +12,10 @@ import { CLIENT_JAVASCRIPT_FIXTURE_SOURCE, assertClientJavaScriptFixtureEvidence, } from "./verify/clients/javascript-fixture.mjs"; +import { + findClientResult, + isClientJavaScriptEvidence, +} from "./verify/clients/result-discovery.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -157,39 +161,21 @@ await withVerifierCleanup(async () => { "name: reverse-engineer-anything", ), ); - const extractEvidence = (value) => { - if (typeof value === "string") { - if ( - value.startsWith("") - ) { - // Hermes wraps external MCP content with its own instruction banner. - // Decode only that observed envelope, retaining the original request artifact. - const start = value.indexOf("\n\n"); - if (start >= 0) - return extractEvidence( - value - .slice(start + 2, value.lastIndexOf("")) - .trim(), - ); - } - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes - ) - return value; - for (const child of Object.values(value)) { - const found = extractEvidence(child); - if (found) return found; - } + const decodeNativeAgentResult = (text) => { + if ( + text.startsWith("") + ) { + // Hermes wraps external MCP content with its own banner; retain original request artifacts. + const start = text.indexOf("\n\n"); + if (start >= 0) + return decodeNativeAgentResult( + text + .slice(start + 2, text.lastIndexOf("")) + .trim(), + ); } + return JSON.parse(text); }; const validateEvidence = async (value) => { const { evidence } = await assertClientJavaScriptFixtureEvidence( @@ -273,7 +259,11 @@ await withVerifierCleanup(async () => { serializedTools.includes(skillPath) && serializedTools.includes("reverse-engineer-anything"); for (const message of toolMessages) { - const evidence = extractEvidence(message.content); + const evidence = findClientResult( + message.content, + isClientJavaScriptEvidence, + decodeNativeAgentResult, + ); if (evidence) await validateEvidence(evidence); } } diff --git a/scripts/verify-opencode-client.mjs b/scripts/verify-opencode-client.mjs index 2273130bfc..acf39a2494 100644 --- a/scripts/verify-opencode-client.mjs +++ b/scripts/verify-opencode-client.mjs @@ -13,6 +13,10 @@ import { CLIENT_JAVASCRIPT_FIXTURE_SOURCE, assertClientJavaScriptFixtureEvidence, } from "./verify/clients/javascript-fixture.mjs"; +import { + findClientResult, + isClientJavaScriptEvidence, +} from "./verify/clients/result-discovery.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -113,27 +117,6 @@ await withVerifierCleanup(async () => { strict: false, validateFormats: false, }); - const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return undefined; - } - } - if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; - } - } - return undefined; - }; fixture = await createOpenAiModelFixture({ directory: lab, prefix: "opencode", @@ -178,7 +161,7 @@ await withVerifierCleanup(async () => { m.content.includes(skillDirectory) && m.content.includes(expectedSkillBody), ); - const observed = extractEvidence(replies); + const observed = findClientResult(replies, isClientJavaScriptEvidence); if (observed) { const { evidence } = await assertClientJavaScriptFixtureEvidence( observed, diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs index 0d05f41ca8..a47e0d6373 100644 --- a/scripts/verify-qwen-client.mjs +++ b/scripts/verify-qwen-client.mjs @@ -19,6 +19,10 @@ import { CLIENT_JAVASCRIPT_FIXTURE_SOURCE, assertClientJavaScriptFixtureEvidence, } from "./verify/clients/javascript-fixture.mjs"; +import { + findClientResult, + isClientJavaScriptEvidence, +} from "./verify/clients/result-discovery.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; @@ -192,32 +196,6 @@ await withVerifierCleanup(async () => { const target = join(lab, "fixture α测试"); await mkdir(target, { recursive: true }); await writeFile(`${target}/app.js`, CLIENT_JAVASCRIPT_FIXTURE_SOURCE); - const extractEvidence = (value) => { - if (typeof value === "string") { - try { - return extractEvidence(JSON.parse(value)); - } catch { - return null; - } - } - if (Array.isArray(value)) { - for (const item of value) { - const result = extractEvidence(item); - if (result) return result; - } - } else if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; - for (const nested of Object.values(value)) { - const result = extractEvidence(nested); - if (result) return result; - } - } - return null; - }; fixture = await createOpenAiModelFixture({ directory: lab, prefix: `${client}-${mode}`, @@ -258,7 +236,6 @@ await withVerifierCleanup(async () => { JSON.parse(artifact.toString("utf8")), target, ); - assert(extractEvidence(evidence)); evidenceSeen = true; await writeFile( `${lab}/${client}-${mode}-evidence.json`, @@ -267,12 +244,14 @@ await withVerifierCleanup(async () => { } for (const message of body.messages ?? []) { if (message.role !== "tool") continue; - const evidence = extractEvidence(message.content); - if (evidence) { - assert( - JSON.stringify(evidence.normalized_result).includes( - "REA_CLIENT_FIXTURE", - ), + const candidate = findClientResult( + message.content, + isClientJavaScriptEvidence, + ); + if (candidate) { + const { evidence } = await assertClientJavaScriptFixtureEvidence( + candidate, + target, ); evidenceSeen = true; await writeFile( diff --git a/scripts/verify/clients/result-discovery.d.mts b/scripts/verify/clients/result-discovery.d.mts new file mode 100644 index 0000000000..7eb7ddb64a --- /dev/null +++ b/scripts/verify/clients/result-discovery.d.mts @@ -0,0 +1,19 @@ +export interface ClientJavaScriptEvidenceCandidate { + readonly evidence_id: string; + readonly normalized_result: { + readonly graph: { readonly nodes: readonly unknown[] }; + }; +} +export function isClientJavaScriptEvidence( + value: unknown, +): value is ClientJavaScriptEvidenceCandidate; +export function findClientResult( + value: unknown, + matches: (value: object) => value is T & object, + decodeText?: (text: string) => unknown, +): (T & object) | undefined; +export function findClientResult( + value: unknown, + matches: (value: object) => boolean, + decodeText?: (text: string) => unknown, +): object | undefined; diff --git a/scripts/verify/clients/result-discovery.mjs b/scripts/verify/clients/result-discovery.mjs new file mode 100644 index 0000000000..2516482dc6 --- /dev/null +++ b/scripts/verify/clients/result-discovery.mjs @@ -0,0 +1,28 @@ +/** Find a result through native JSON containers without hiding assertion faults. */ +export const findClientResult = (value, matches, decodeText = JSON.parse) => { + if (typeof value === "string") { + let decoded; + try { + decoded = decodeText(value); + } catch (cause) { + if (cause instanceof SyntaxError) return undefined; + throw cause; + } + return findClientResult(decoded, matches, decodeText); + } + if (value !== null && typeof value === "object") { + if (matches(value)) return value; + for (const nested of Object.values(value)) { + const result = findClientResult(nested, matches, decodeText); + if (result !== undefined) return result; + } + } + return undefined; +}; + +/** Empty graphs remain candidates: the assertion owner must reject bad fixtures. */ +export const isClientJavaScriptEvidence = (value) => + value !== null && + typeof value === "object" && + typeof value.evidence_id === "string" && + Array.isArray(value.normalized_result?.graph?.nodes); diff --git a/tests/conformance/verifiers/clientResultDiscovery.test.ts b/tests/conformance/verifiers/clientResultDiscovery.test.ts new file mode 100644 index 0000000000..3342d4ccab --- /dev/null +++ b/tests/conformance/verifiers/clientResultDiscovery.test.ts @@ -0,0 +1,89 @@ +import { expect, it } from "vitest"; + +import { + findClientResult, + isClientJavaScriptEvidence, +} from "../../../scripts/verify/clients/result-discovery.mjs"; + +const candidate = (nodes: unknown[] = [{}]) => ({ + evidence_id: "fixture-evidence", + normalized_result: { graph: { nodes } }, +}); + +it("finds Evidence through nested arrays, objects and serialized JSON", () => { + const evidence = candidate(); + const messages = [ + { content: "not JSON" }, + { content: JSON.stringify({ result: [JSON.stringify(evidence)] }) }, + ]; + expect(findClientResult(messages, isClientJavaScriptEvidence)).toEqual( + evidence, + ); + expect(findClientResult(null, isClientJavaScriptEvidence)).toBeUndefined(); + expect(findClientResult("null", isClientJavaScriptEvidence)).toBeUndefined(); + expect( + findClientResult("{broken", isClientJavaScriptEvidence), + ).toBeUndefined(); +}); + +it("returns the first empty graph for fixture assertions instead of skipping it", () => { + const empty = candidate([]); + expect( + findClientResult([empty, candidate()], isClientJavaScriptEvidence), + ).toBe(empty); + for (const invalid of [undefined, null, "nodes", {}, { length: 1 }]) { + expect( + isClientJavaScriptEvidence({ + evidence_id: "fixture-evidence", + normalized_result: { graph: { nodes: invalid } }, + }), + ).toBe(false); + } + expect( + isClientJavaScriptEvidence({ normalized_result: { graph: { nodes: [] } } }), + ).toBe(false); +}); + +it("uses the selected text decoder at every nested level", () => { + const evidence = candidate(); + const decoded: string[] = []; + const decodeNativeEnvelope = (text: string): unknown => { + decoded.push(text); + return JSON.parse( + text.startsWith("\n") && text.endsWith("\n") + ? text.slice("\n".length, -"\n".length) + : text, + ); + }; + const inner = `\n${JSON.stringify(evidence)}\n`; + const outer = `\n${JSON.stringify({ result: [inner] })}\n`; + expect( + findClientResult(outer, isClientJavaScriptEvidence, decodeNativeEnvelope), + ).toEqual(evidence); + expect(decoded).toEqual([outer, inner]); + expect( + findClientResult( + "{}", + isClientJavaScriptEvidence, + decodeNativeEnvelope, + ), + ).toBeUndefined(); +}); + +it("propagates predicate faults even when they are SyntaxErrors", () => { + const failure = new SyntaxError("assertion failed"); + expect(() => + findClientResult(JSON.stringify(candidate()), () => { + throw failure; + }), + ).toThrow(failure); +}); + +it("propagates decoder failures outside JSON syntax errors", () => { + const failure = new Error("native decoder failed"); + expect(() => + findClientResult("body", isClientJavaScriptEvidence, () => { + throw failure; + }), + ).toThrow(failure); +});