diff --git a/scripts/verify-il2cpp-metadata.mjs b/scripts/verify-il2cpp-metadata.mjs index 362208cd09..ac73c7f951 100644 --- a/scripts/verify-il2cpp-metadata.mjs +++ b/scripts/verify-il2cpp-metadata.mjs @@ -79,6 +79,24 @@ await withVerifierCleanup(async () => { assert.equal(error.code, "resource_constraint"); assert.equal(error.details.reported_limits.boundary, "mcp-response"); assert.match(error.details.reported_limits.evidence_id, /^ev_/u); + const retained = ( + await call("inspect_analysis_view", { + source: { + kind: "retained-evidence", + evidence_id: error.details.reported_limits.evidence_id, + }, + view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 }, + }) + ).normalized_result; + assert.ok( + report.methods.length > 0, + "Overflow retention needs a nonempty method oracle", + ); + assert.deepEqual( + retained.items, + report.methods.slice(0, 16).map((row, index) => ({ ...row, index })), + "Failed complete delivery must retain its exact Evidence before reanalysis", + ); console.log( `[ok] MCP complete delivery refused at the stdio budget (${error.details.reported_limits.response_bytes_at_least} bytes) with retained Evidence`, ); diff --git a/scripts/verify-native-ui.mjs b/scripts/verify-native-ui.mjs index b7bfb43536..ebc28c883c 100644 --- a/scripts/verify-native-ui.mjs +++ b/scripts/verify-native-ui.mjs @@ -165,7 +165,9 @@ try { scenario.value.steps.some((step) => step.outcome !== "completed") || !scenario.value.steps .at(-1) - ?.after?.nodes.some((node) => node.title === "REA fixture incremented") + ?.after?.nodes.some( + (node) => node.title === "REA fixture incremented 1", + ) ) throw new Error( `Source-owned AX scenario failed: ${JSON.stringify(scenario.value.steps.map(({ outcome, reason }) => ({ outcome, reason })))}`, @@ -195,7 +197,12 @@ try { evidence.normalized_result?.steps?.some( (step) => step.outcome !== "completed", ) || - evidence.normalized_result?.steps?.length !== 2 + evidence.normalized_result?.steps?.length !== 2 || + !evidence.normalized_result?.steps + ?.at(-1) + ?.after?.nodes.some( + (node) => node.title === "REA fixture incremented 2", + ) ) throw new Error( `Native UI CLI did not preserve the selected scenario: ${JSON.stringify(evidence.error ?? evidence.normalized_result?.steps?.map(({ outcome, reason }) => ({ outcome, reason })))}`, @@ -232,7 +239,7 @@ try { !captured.steps .at(-1) ?.after?.nodes.some( - (node) => node.title === "REA fixture incremented", + (node) => node.title === "REA fixture incremented 3", ) ) throw new Error("MCP did not observe the selected fixture's AX action"); diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs index 6011c6d711..58fd63c3d6 100644 --- a/scripts/verify-qwen-client.mjs +++ b/scripts/verify-qwen-client.mjs @@ -310,7 +310,6 @@ await withVerifierCleanup(async () => { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK", }; - finalSeen = true; } else if (skill && !skillRequested) { skillRequested = true; message = call(skill.name, { skill: "reverse-engineer-anything" }); @@ -401,11 +400,13 @@ await withVerifierCleanup(async () => { stderr: join(lab, `${mode}.stderr`), }, ); - const initialized = response.stdout + const events = response.stdout .split("\n") .filter(Boolean) - .map((line) => JSON.parse(line)) - .find((v) => v.type === "system" && v.subtype === "init"); + .map((line) => JSON.parse(line)); + const initialized = events.find( + (v) => v.type === "system" && v.subtype === "init", + ); assert(initialized); const observed = initialized.tools .filter((n) => n.startsWith("mcp__rea__")) @@ -417,7 +418,15 @@ await withVerifierCleanup(async () => { "Qwen must discover the complete REA catalog", ); assert.equal(fixture.failure, undefined); - assert(finalSeen); + const result = events.findLast((event) => event.type === "result"); + assert.equal( + result?.subtype, + "success", + "Qwen must emit a successful terminal result", + ); + assert.equal(result.is_error, false); + assert.equal(result.result, "REA_CLIENT_COMPATIBILITY_OK"); + finalSeen = true; if (mode === "call") { assert(evidenceSeen); assert(selectedSkillLoaded); diff --git a/scripts/verify/managed/oracles.mjs b/scripts/verify/managed/oracles.mjs index 6f717fd8a6..8d90e0e472 100644 --- a/scripts/verify/managed/oracles.mjs +++ b/scripts/verify/managed/oracles.mjs @@ -32,50 +32,61 @@ async function runManagedAppManifestSelfTest(context) { const members = inspectManagedMembersBytes(sample.bytes, sample.target); const method = members.methods[0]; assert.ok(method); - const summary = await context.verifyManagedAppManifest( - { - label: "source-owned manifest verifier self-test", - target: { - path: sample.path, - sha256: sample.target.sha256, - mvid: artifact.module?.mvid, - assembly_name: artifact.assembly?.name, - runtime_family: artifact.classification.runtime_family, - managed_architecture: artifact.classification.managed_architecture, + const manifest = { + label: "source-owned manifest verifier self-test", + target: { + path: sample.path, + sha256: sample.target.sha256, + mvid: artifact.module?.mvid, + assembly_name: artifact.assembly?.name, + runtime_family: artifact.classification.runtime_family, + managed_architecture: artifact.classification.managed_architecture, + }, + methods: [ + { + label: "pinned-semantic-slice", + token: method.token, + signature_sha256: method.signature.raw_sha256, + il_size: method.body.il_size, + il_sha256: method.body.il_sha256, + normalized_il_sha256: method.body.normalized_il_sha256, }, - methods: [ + ], + application_graph: { + expected_node_kinds: [ + "artifact", + "managed-assembly", + "managed-module", + "managed-type", + "managed-method", + ], + feature_traces: [ { - label: "pinned-semantic-slice", - token: method.token, - signature_sha256: method.signature.raw_sha256, - il_size: method.body.il_size, - il_sha256: method.body.il_sha256, - normalized_il_sha256: method.body.normalized_il_sha256, + label: "trace pinned method by name", + method_token: method.token, + seed: "PinnedSemanticSlice", + match: "exact", + min_matched_seeds: 1, }, ], - application_graph: { - expected_node_kinds: [ - "artifact", - "managed-assembly", - "managed-module", - "managed-type", - "managed-method", - ], - feature_traces: [ - { - label: "trace pinned method by name", - method_token: method.token, - seed: "PinnedSemanticSlice", - match: "exact", - min_matched_seeds: 1, - }, - ], - }, }, - join(context.workspace, "source-owned-managed-app-manifest.json"), + }; + const manifestPath = join( + context.workspace, + "source-owned-managed-app-manifest.json", + ); + const summary = await context.verifyManagedAppManifest( + manifest, + manifestPath, + ); + const rejectedMismatches = await verifyManifestRejections( + context, + manifest, + manifestPath, ); return { verified: true, + rejected_mismatches: rejectedMismatches, assertions: summary.assertions, methods: summary.methods.length, applicationGraph: summary.application_graph ?? null, @@ -83,6 +94,60 @@ async function runManagedAppManifestSelfTest(context) { }; } +async function verifyManifestRejections(context, manifest, manifestPath) { + const refusals = [ + [ + "target sha256", + ["target", "sha256"], + "0".repeat(64), + /target sha256 mismatch/u, + ], + [ + "method token", + ["methods", 0, "token"], + "0x0600ffff", + /method 0x0600ffff was not returned/u, + ], + [ + "signature sha256", + ["methods", 0, "signature_sha256"], + "0".repeat(64), + /signature sha256 mismatch/u, + ], + ["IL size", ["methods", 0, "il_size"], 999, /IL size mismatch/u], + [ + "IL sha256", + ["methods", 0, "il_sha256"], + "0".repeat(64), + /IL sha256 mismatch/u, + ], + [ + "normalized IL sha256", + ["methods", 0, "normalized_il_sha256"], + "0".repeat(64), + /normalized IL sha256 mismatch/u, + ], + ]; + for (const [label, path, wrong, expected] of refusals) { + const rejected = structuredClone(manifest); + let owner = rejected; + for (const key of path.slice(0, -1)) owner = owner[key]; + const key = path.at(-1); + assert.notEqual( + owner[key], + wrong, + `${label} control must differ from the valid manifest`, + ); + owner[key] = wrong; + await assert.rejects( + () => context.verifyManagedAppManifest(rejected, manifestPath), + expected, + `Manifest verifier accepted an authored wrong ${label}`, + ); + } + return refusals.map(([label]) => label); +} + async function runOptionalManagedAppManifest(context) { const manifestPath = process.env.REA_MANAGED_APP_MANIFEST_PATH; if (manifestPath === undefined) return null; diff --git a/tests/boundary/process/il2cppVerifierRetention.test.ts b/tests/boundary/process/il2cppVerifierRetention.test.ts new file mode 100644 index 0000000000..811348a9f2 --- /dev/null +++ b/tests/boundary/process/il2cppVerifierRetention.test.ts @@ -0,0 +1,97 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +const execute = promisify(execFile); + +it.each([true, false])( + "requires overflow Evidence to be retained before another analysis (retained=%s)", + async (retained) => { + const directory = await mkdtemp( + join(tmpdir(), "rea-il2cpp-retention-oracle-"), + ); + try { + // This recording qualifies the verifier's ordering oracle, not metadata + // decoding or transport conformance. A later analysis repairs retention. + await writeFile( + join(directory, "recording.mjs"), + ` +import {promisify} from 'node:util'; +const path=process.env.INPUT; +const identity={sha256:'a'.repeat(64),bytes:1}; +const report={artifact:{path,...identity},magic:0xfab11baf,sections:[],images:[{type_count:1}],types:[{declaring_type:-1,method_start:0,method_count:1}],methods:[{declaring_type:0,name:'AuthoredMethod'}],fields:[],properties:[],events:[],parameters:[],string_literals:[]}; +const counts={sections:0,images:1,types:1,methods:1,nested_types:0,fields:0,properties:0,events:0,parameters:0,string_literals:0,...Object.fromEntries(['code_gen_modules','method_addresses','method_address_states','return_types','field_types','parameter_types','generic_method_instances','generic_method_instance_states','generic_method_issues'].map(k=>[k,null]))}; +let reanalyzed=false; +export class Client { + async connect(){} async close(){} async listTools(){return {tools:[{name:'inspect_il2cpp_metadata'}]};} + async callTool({name,arguments:input}){ + if(name==='inspect_il2cpp_metadata'&&input.detail==='complete')return {isError:true,error:{code:'resource_constraint',details:{reported_limits:{boundary:'mcp-response',evidence_id:'ev_authored_parent',response_bytes_at_least:999}}}}; + if(name==='inspect_il2cpp_metadata'){reanalyzed=true;return {structuredContent:{normalized_result:{summary:{magic:report.magic,counts},parent_evidence_id:'ev_authored_parent'}}};} + if(!reanalyzed&&process.env.RETAINED==='false')throw Error('Failed delivery Evidence was not retained before reanalysis'); + if(input.source.evidence_id!=='ev_authored_parent')throw Error('Wrong parent Evidence'); + const rows=report[input.view.collection],offset=input.view.offset??0; + return {structuredContent:{normalized_result:{items:rows.slice(offset,offset+(input.view.limit??16)).map((row,index)=>({...row,index:offset+index})),coverage:{next_offset:null}}}}; + } +} +export class StdioClientTransport {async close(){} } +export const execFile=Object.assign(()=>{}, {[promisify.custom]:async()=>({stdout:JSON.stringify({normalized_result:report})})}); +export const toolContract=()=>({outputSchema:{parse:value=>value}}); +export const requireMcpOutput=response=>{if(response.isError)throw Error('Unexpected failed tool');return response.structuredContent;}; +export const requireMcpToolError=response=>response.error; +export const hashVerifierFile=async()=>identity; +`, + ); + await writeFile( + join(directory, "hook.mjs"), + ` +import {registerHooks} from 'node:module';import {pathToFileURL} from 'node:url'; +registerHooks({resolve(specifier,context,next){ + if(context.parentURL?.endsWith('/scripts/verify-il2cpp-metadata.mjs')&&['node:child_process','@modelcontextprotocol/client','@modelcontextprotocol/client/stdio','../dist/contracts/toolContracts.js','./lib/mcp-verifier-results.mjs','./lib/verifier-file-resources.mjs'].includes(specifier))return {shortCircuit:true,url:pathToFileURL(process.env.RECORDING).href}; + return next(specifier,context); +}}); +`, + ); + const result = execute( + process.execPath, + [ + "--import", + join(directory, "hook.mjs"), + fileURLToPath( + new URL( + "../../../scripts/verify-il2cpp-metadata.mjs", + import.meta.url, + ), + ), + "--metadata", + join(directory, "metadata.dat"), + ], + { + timeout: 10_000, + env: { + ...process.env, + RECORDING: join(directory, "recording.mjs"), + INPUT: join(directory, "metadata.dat"), + RETAINED: String(retained), + }, + }, + ); + if (retained) { + expect((await result).stdout).toContain( + "All asserted IL2CPP metadata CLI/MCP checks passed", + ); + } else { + await expect(result).rejects.toMatchObject({ + stderr: expect.stringContaining( + "Failed delivery Evidence was not retained before reanalysis", + ), + }); + } + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, +); diff --git a/tests/boundary/process/nativeUiVerifierActions.test.ts b/tests/boundary/process/nativeUiVerifierActions.test.ts new file mode 100644 index 0000000000..7f6ff9bfd7 --- /dev/null +++ b/tests/boundary/process/nativeUiVerifierActions.test.ts @@ -0,0 +1,89 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +const execute = promisify(execFile); + +it.each(["complete", "drop CLI", "drop MCP"])( + "requires each native UI adapter's own action (%s)", + async (mode) => { + const directory = await mkdtemp(join(tmpdir(), "rea-ui-action-oracle-")); + try { + // Authored AX recordings qualify adapter action assertions, not macOS + // permissions, native capture, Swift compilation or process ownership. + await writeFile( + join(directory, "recording.mjs"), + ` +import {EventEmitter} from 'node:events';import {writeFile} from 'node:fs/promises';import {promisify} from 'node:util'; +let count=0; +const nodes=()=>[{role:'AXButton',path:[0],title:count===0?'Increment REA fixture':'REA fixture incremented '+count}]; +const captured=()=>({steps:[{outcome:'completed'},{outcome:'completed',after:{nodes:nodes()}}]}); +export const execFile=Object.assign(()=>{}, {[promisify.custom]:async (file,args)=>{ + if(args?.includes('capture-native-ui-scenario')){if(process.env.MODE!=='drop CLI')count++;return {stdout:JSON.stringify({normalized_result:captured()})};} + return {stdout:'{"v":"slash/producer"}\\n{"v":1.0}\\n'}; +}}); +export const spawn=(_command,args)=>{const child=new EventEmitter();child.exitCode=null;child.kill=()=>{child.exitCode=0;child.emit('exit',0);return true;};void writeFile(args[args.indexOf('--stdout')+1],JSON.stringify({pid:424242,window_id:7})+'\\n');return child;}; +export const parseBinaryTarget=async path=>({ok:true,value:{path,sha256:'a'.repeat(64)}}); +export const observeProcessStartIdentity=async()=>({state:'readable',identity:'authored-process-identity'}); +export const observeNativeUi=async(target,operation)=>{ + if(target.sha256==='0'.repeat(64))return {ok:false,error:{issues:[{message:'target-mismatch: authored changed digest'}]}}; + if(operation==='capture_native_ui_scenario'){count++;return {ok:true,value:captured()};} + return {ok:true,value:{initial:{nodes:nodes()}}}; +}; +export const withArtifactMcp=async(_path,run)=>run({}); +export const artifactMcpResult=async(_client,name)=>{if(name==='observe_native_ui')return {initial:{nodes:nodes()}};if(process.env.MODE!=='drop MCP')count++;return captured();}; +`, + ); + await writeFile( + join(directory, "hook.mjs"), + ` +import {registerHooks} from 'node:module';import {pathToFileURL} from 'node:url'; +Object.defineProperty(process,'platform',{value:'darwin'});process.kill=()=>true; +registerHooks({resolve(specifier,context,next){if(context.parentURL?.endsWith('/scripts/verify-native-ui.mjs')&&['node:child_process','../dist/application/BinaryTargetResolver.js','../dist/native/NativeUiObservation.js','./lib/artifact-e2e.mjs','../dist/process/ProcessOwnershipObservation.js'].includes(specifier))return {shortCircuit:true,url:pathToFileURL(process.env.RECORDING).href};return next(specifier,context);}}); +`, + ); + const result = execute( + process.execPath, + [ + "--import", + join(directory, "hook.mjs"), + fileURLToPath( + new URL("../../../scripts/verify-native-ui.mjs", import.meta.url), + ), + ], + { + timeout: 10_000, + env: { + ...process.env, + TMPDIR: directory, + TMP: directory, + TEMP: directory, + RECORDING: join(directory, "recording.mjs"), + MODE: mode, + }, + }, + ); + if (mode === "complete") { + expect(JSON.parse((await result).stdout)).toMatchObject({ + ok: true, + positive_e2e: true, + scenario: "selected-AX-button-action-and-CLI-MCP-parity-observed", + }); + } else { + await expect(result).rejects.toMatchObject({ + stderr: expect.stringContaining( + mode === "drop CLI" + ? "Native UI CLI did not preserve the selected scenario" + : "MCP did not observe the selected fixture's AX action", + ), + }); + } + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, +); diff --git a/tests/boundary/process/qwenVerifierSkillBody.test.ts b/tests/boundary/process/qwenVerifierSkillBody.test.ts index e0d2ab6b99..f6ef708561 100644 --- a/tests/boundary/process/qwenVerifierSkillBody.test.ts +++ b/tests/boundary/process/qwenVerifierSkillBody.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -13,12 +13,16 @@ it.each([ ["split", false], ["complete", true], ["CRLF", true], + ["output missing result", false], + ["output missing marker", false], + ["output terminal failure", false], + ["output success", true], ] as const)("binds Qwen skill evidence for %s", async (mode, loaded) => { const state = await mkdtemp(join(tmpdir(), "rea-qwen-skill-")); try { await writeFile( join(state, "commands.mjs"), - "import {mkdir,readFile,writeFile} from 'node:fs/promises';import {join} from 'node:path';\nexport async function createVerifierCommands(){return {async exec(command,args,options={}){\n if(args.includes('--version'))return {stdout:'fake',stderr:''};\n const profile=options.env.QWEN_HOME;const config=join(profile,'settings.json');\n if(args.includes('setup')&&args.includes('--dry-run'))return {stdout:JSON.stringify({plannedActions:[{target:config}]}),stderr:''};\n if(args.includes('setup')){\n const original=await readFile(config,'utf8');await writeFile(config+'.rea.backup',original);\n const value=JSON.parse(original);value.mcpServers.rea={command:'/fake/rea',args:['mcp']};await writeFile(config,JSON.stringify(value));\n const skill=join(options.env.HOME,'.agents','skills','reverse-engineer-anything');await mkdir(skill,{recursive:true});\n const source='---\\nname: reverse-engineer-anything\\n---\\nQwen \"authored\" body\\nKeep a \\\\ literal\\n';\n await writeFile(join(skill,'SKILL.md'),process.env.MODE==='CRLF'?source.replaceAll('\\n','\\r\\n'):source);\n return {stdout:'{}',stderr:''};\n }\n if(args.includes('doctor'))return {stdout:'{}',stderr:''};\n const skill=join(options.env.HOME,'.agents','skills','reverse-engineer-anything');\n const content='Base directory for this skill: '+skill+'\\nImportant: ALWAYS resolve absolute paths from this base directory when working with skills.\\n\\nQwen \"authored\" body\\nKeep a \\\\ literal\\n';\n const tools=[...(process.env.MODE==='no skill tool'?[]:[{name:'skill'}]),{name:'mcp__rea__analyze_javascript_application'}];\n const first=await globalThis.fakeRequest({body:{messages:[]},tools,requestIndex:1});const callId=first.tool_calls[0].id;\n let messages=(process.env.MODE==='user mention'||process.env.MODE==='no skill tool')?[{role:'user',content:'Unrelated '+skill}]:[{role:'tool',tool_call_id:process.env.MODE==='wrong id'?'other':callId,content:process.env.MODE==='missing body'?'Base directory for this skill: '+skill:content}];\n if(process.env.MODE==='split')messages=[{role:'tool',tool_call_id:callId,content:'Base directory for this skill: '+skill},{role:'tool',tool_call_id:'other',content}];\n const second=await globalThis.fakeRequest({body:{messages},tools,requestIndex:2});await writeFile(process.env.STATE+'/observed.json',JSON.stringify({first,second,messages}));throw Error('intentional stop before client output');\n},async close(){}};}\n", + "import {mkdir,readFile,writeFile} from 'node:fs/promises';import {join} from 'node:path';\nexport async function createVerifierCommands(){return {async exec(command,args,options={}){\n if(args.includes('--version'))return {stdout:'fake',stderr:''};\n const profile=options.env.QWEN_HOME;const config=join(profile,'settings.json');\n if(args.includes('setup')&&args.includes('--dry-run'))return {stdout:JSON.stringify({plannedActions:[{target:config}]}),stderr:''};\n if(args.includes('setup')){\n const original=await readFile(config,'utf8');await writeFile(config+'.rea.backup',original);\n const value=JSON.parse(original);value.mcpServers.rea={command:'/fake/rea',args:['mcp']};await writeFile(config,JSON.stringify(value));\n const skill=join(options.env.HOME,'.agents','skills','reverse-engineer-anything');await mkdir(skill,{recursive:true});\n const source='---\\nname: reverse-engineer-anything\\n---\\nQwen \"authored\" body\\nKeep a \\\\ literal\\n';\n await writeFile(join(skill,'SKILL.md'),process.env.MODE==='CRLF'?source.replaceAll('\\n','\\r\\n'):source);\n return {stdout:'{}',stderr:''};\n }\n if(args.includes('doctor'))return {stdout:'{}',stderr:''};\n if(process.env.MODE.startsWith('output ')){await globalThis.fakeRequest({body:{messages:[]},tools:[],requestIndex:1});const {TOOL_CONTRACTS}=await import(process.env.CONTRACTS);const events=[{type:'system',subtype:'init',tools:TOOL_CONTRACTS.map(c=>'mcp__rea__'+c.name)}];const success={type:'result',subtype:'success',is_error:false,result:process.env.MODE==='output missing marker'?'': 'REA_CLIENT_COMPATIBILITY_OK'};if(process.env.MODE!=='output missing result')events.push(success);if(process.env.MODE==='output terminal failure')events.push({type:'result',subtype:'error_during_execution',is_error:true});return {stdout:events.map(e=>JSON.stringify(e)).join('\\n')+'\\n',stderr:''};}\n const skill=join(options.env.HOME,'.agents','skills','reverse-engineer-anything');\n const content='Base directory for this skill: '+skill+'\\nImportant: ALWAYS resolve absolute paths from this base directory when working with skills.\\n\\nQwen \"authored\" body\\nKeep a \\\\ literal\\n';\n const tools=[...(process.env.MODE==='no skill tool'?[]:[{name:'skill'}]),{name:'mcp__rea__analyze_javascript_application'}];\n const first=await globalThis.fakeRequest({body:{messages:[]},tools,requestIndex:1});const callId=first.tool_calls[0].id;\n let messages=(process.env.MODE==='user mention'||process.env.MODE==='no skill tool')?[{role:'user',content:'Unrelated '+skill}]:[{role:'tool',tool_call_id:process.env.MODE==='wrong id'?'other':callId,content:process.env.MODE==='missing body'?'Base directory for this skill: '+skill:content}];\n if(process.env.MODE==='split')messages=[{role:'tool',tool_call_id:callId,content:'Base directory for this skill: '+skill},{role:'tool',tool_call_id:'other',content}];\n const second=await globalThis.fakeRequest({body:{messages},tools,requestIndex:2});await writeFile(process.env.STATE+'/observed.json',JSON.stringify({first,second,messages}));throw Error('intentional stop before client output');\n},async close(){}};}\n", ); await writeFile( join(state, "fixture.mjs"), @@ -33,7 +37,12 @@ it.each([ ); const result = spawnSync( process.execPath, - ["--import", join(state, "hook.mjs"), script], + [ + "--import", + join(state, "hook.mjs"), + script, + ...(mode.startsWith("output ") ? ["chat"] : []), + ], { encoding: "utf8", timeout: 10_000, @@ -41,12 +50,25 @@ it.each([ ...process.env, STATE: state, MODE: mode, + CONTRACTS: new URL( + "../../../dist/contracts/toolContracts.js", + import.meta.url, + ).href, TMPDIR: state, TMP: state, TEMP: state, }, }, ); + if (mode.startsWith("output ")) { + // Authored stream result shape from Qwen's BaseJsonOutputAdapter at + // QwenLM/qwen-code c238beb56fa3c65a0f62ad7a3b2cd14db978b998. + expect(result.status, result.stderr).toBe(loaded ? 0 : 1); + const receipt = JSON.parse(result.stdout.trim().split("\n")[0] ?? ""); + expect(receipt.finalSeen).toBe(loaded); + expect(receipt.passed).toBe(loaded); + return; + } expect(result.status).not.toBe(0); expect(result.stderr).toContain("intentional stop before client output"); const receipt = JSON.parse(result.stdout.trim().split("\n")[0] ?? ""); diff --git a/tests/conformance/native/ui.swift b/tests/conformance/native/ui.swift index 1d399f00de..6a2b13f31a 100644 --- a/tests/conformance/native/ui.swift +++ b/tests/conformance/native/ui.swift @@ -6,7 +6,11 @@ let window = NSWindow(contentRect: NSRect(x: 120, y: 120, width: 360, height: 18 window.title = "REA source-owned UI verification" window.animationBehavior = .none final class FixtureController: NSObject { - @objc func increment(_ sender: NSButton) { sender.title = "REA fixture incremented" } + private var count = 0 + @objc func increment(_ sender: NSButton) { + count += 1 + sender.title = "REA fixture incremented \(count)" + } } let controller = FixtureController() let button = NSButton(frame: NSRect(x: 30, y: 60, width: 280, height: 40))