diff --git a/docs/architecture/eval-harness-frameworks.md b/docs/architecture/eval-harness-frameworks.md index d00e4b4e0..d483170b6 100644 --- a/docs/architecture/eval-harness-frameworks.md +++ b/docs/architecture/eval-harness-frameworks.md @@ -313,6 +313,18 @@ an implementation. The legacy `EFFECT_FENCE_PRELOAD` export remains for import compatibility, but loading that file always throws `gate.isolation_required`. It no longer attempts JavaScript interception. +Record-mode calls preserve synchronous results. If an implementation returns a +Promise or thenable, await `call()` before consuming the result: the fixture and +`recorded` trace are written only after fulfillment. An implementation rejection +propagates without writing a fixture or successful trace and leaves any previous +fixture unchanged. +Replay remains synchronous and never invokes the implementation. +Fixture keys use a JSON snapshot of the arguments at invocation, before the +implementation runs; later argument mutations cannot redirect the recording. +Re-record fixtures previously captured from async implementations: an empty +object recorded from an unresolved Promise still has a valid hash and cannot +be distinguished automatically from a real empty response. + ## Offline receipts ```sh diff --git a/scripts/lib/eval-harness/replay.js b/scripts/lib/eval-harness/replay.js index 509c5ae97..6a8d1faf8 100644 --- a/scripts/lib/eval-harness/replay.js +++ b/scripts/lib/eval-harness/replay.js @@ -86,6 +86,7 @@ class FixtureStore { /** * tools: { name: { effect_class, determinism: 'deterministic'|'nondeterministic', impl(args) } } * options: { mode: 'record'|'replay', store: FixtureStore, maxEffectClass: 'SE2', onCall(entry) } + * Record calls return a Promise only when the implementation returns a thenable. */ function createReplayer(tools, options = {}) { const mode = options.mode || 'replay'; @@ -136,10 +137,20 @@ function createReplayer(tools, options = {}) { emit({ tool: name, mode, status: 'replayed', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash }); return record.response; } + // Keep fixture identity at invocation time, even if the caller or the + // implementation changes a nested argument before an async result settles. + const invocationArgs = JSON.parse(canonicalJson(args)); const response = tool.impl(args); - const record = store.put(name, args, response); - emit({ tool: name, mode, status: 'recorded', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash }); - return response; + const recordResponse = (resolved) => { + const record = store.put(name, invocationArgs, resolved); + emit({ tool: name, mode, status: 'recorded', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash }); + return resolved; + }; + // Persist settled values, never the Promise object or a failed request. + if (response && typeof response.then === 'function') { + return Promise.resolve(response).then(recordResponse); + } + return recordResponse(response); }, }; } diff --git a/tests/lib/eval-harness/helpers.js b/tests/lib/eval-harness/helpers.js index fc201ed74..a760c8950 100644 --- a/tests/lib/eval-harness/helpers.js +++ b/tests/lib/eval-harness/helpers.js @@ -20,6 +20,28 @@ function test(name, fn) { } } +async function asyncTest(name, fn, timeoutMs = 5000) { + let timer; + try { + // Keep a referenced timer: an unresolved Promise alone lets Node exit 0 + // before finish() can report an incomplete test as a failure. + await Promise.race([ + Promise.resolve().then(fn), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs); + }) + ]); + console.log(` ✓ ${name}`); + passed += 1; + } catch (error) { + console.log(` ✗ ${name}`); + console.log(` Error: ${error.message}`); + failed += 1; + } finally { + clearTimeout(timer); + } +} + function tempDir(prefix) { return fs.mkdtempSync(path.join(os.tmpdir(), `ecc-eval-harness-${prefix}-`)); } @@ -55,4 +77,4 @@ function runNpm(args, options = {}) { const fixedClock = () => new Date('2026-09-02T00:00:00.000Z'); -module.exports = { test, tempDir, cleanup, finish, fixedClock, runNpm }; +module.exports = { test, asyncTest, tempDir, cleanup, finish, fixedClock, runNpm }; diff --git a/tests/lib/eval-harness/replay-async.test.js b/tests/lib/eval-harness/replay-async.test.js new file mode 100644 index 000000000..a7e5f8337 --- /dev/null +++ b/tests/lib/eval-harness/replay-async.test.js @@ -0,0 +1,170 @@ +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const replay = require('../../../scripts/lib/eval-harness/replay'); +const { asyncTest, tempDir, cleanup, finish } = require('./helpers'); + +async function main() { + console.log('\n=== eval-harness async replay ===\n'); + + await asyncTest('record waits for an async tool before persisting its resolved response', async () => { + const dir = tempDir('async-record'); + try { + const store = new replay.FixtureStore(dir); + const args = { sku: 'x' }; + const response = { sku: 'x', count: 7 }; + const trace = []; + let complete; + let liveCalls = 0; + const tools = { read_inventory: { + effect_class: 'SE0', determinism: 'deterministic', + impl() { liveCalls += 1; return new Promise(resolve => { complete = resolve; }); } + } }; + const recorder = replay.createReplayer(tools, { mode: 'record', store, onCall: entry => trace.push(entry) }); + const pending = recorder.call('read_inventory', args); + + assert.strictEqual(store.has('read_inventory', args), false, 'Pending response is not a fixture'); + assert.deepStrictEqual(recorder.calls, [], 'Pending response is not a successful call'); + assert.deepStrictEqual(trace, []); + complete(response); + + assert.deepStrictEqual(await pending, response); + assert.deepStrictEqual(store.get('read_inventory', args).response, response); + assert.strictEqual(recorder.calls.length, 1); + assert.strictEqual(recorder.calls[0].status, 'recorded'); + assert.deepStrictEqual(trace, recorder.calls); + + const replayer = replay.createReplayer(tools, { mode: 'replay', store }); + assert.deepStrictEqual(replayer.call('read_inventory', args), response); + assert.strictEqual(liveCalls, 1, 'Replay must not call the live async implementation'); + } finally { cleanup(dir); } + }); + + await asyncTest('async rejection preserves the error without creating a successful fixture or trace', async () => { + const dir = tempDir('async-rejection'); + try { + const store = new replay.FixtureStore(dir); + const failure = new Error('inventory request failed'); + const trace = []; + const tools = { read_inventory: { + effect_class: 'SE0', determinism: 'nondeterministic', + async impl() { throw failure; } + } }; + const recorder = replay.createReplayer(tools, { mode: 'record', store, onCall: entry => trace.push(entry) }); + await assert.rejects(recorder.call('read_inventory', {}), error => error === failure); + assert.deepStrictEqual(fs.readdirSync(dir), []); + assert.deepStrictEqual(recorder.calls, []); + assert.deepStrictEqual(trace, []); + const replayer = replay.createReplayer(tools, { mode: 'replay', store }); + assert.throws(() => replayer.call('read_inventory', {}), error => error.code === 'tool.fixture_missing'); + } finally { cleanup(dir); } + }); + + await asyncTest('a failed async refresh leaves an existing fixture unchanged', async () => { + const dir = tempDir('async-refresh'); + try { + const store = new replay.FixtureStore(dir); + const args = { sku: 'x' }; + store.put('read_inventory', args, { count: 7 }); + const before = fs.readFileSync(store.pathFor(store.key('read_inventory', args))); + const tools = { read_inventory: { + effect_class: 'SE0', determinism: 'nondeterministic', + async impl() { throw new Error('offline'); } + } }; + const recorder = replay.createReplayer(tools, { mode: 'record', store }); + await assert.rejects(recorder.call('read_inventory', args), /offline/); + assert.deepStrictEqual(fs.readFileSync(store.pathFor(store.key('read_inventory', args))), before); + assert.deepStrictEqual(recorder.calls, []); + } finally { cleanup(dir); } + }); + + await asyncTest('record assimilates thenable tool results before hashing', async () => { + const dir = tempDir('thenable'); + try { + const store = new replay.FixtureStore(dir); + const response = { count: 7 }; + const tools = { read_inventory: { + effect_class: 'SE0', determinism: 'deterministic', + impl() { return { then(resolve) { resolve(response); } }; } + } }; + const recorder = replay.createReplayer(tools, { mode: 'record', store }); + assert.deepStrictEqual(await recorder.call('read_inventory', {}), response); + assert.deepStrictEqual(store.get('read_inventory', {}).response, response); + assert.strictEqual(recorder.calls[0].status, 'recorded'); + } finally { cleanup(dir); } + }); + + for (const mutateInImplementation of [false, true]) { + await asyncTest(`records invocation arguments despite ${mutateInImplementation ? 'implementation' : 'caller'} mutation`, async () => { + const dir = tempDir('async-arguments'); + try { + const store = new replay.FixtureStore(dir); + const args = { query: { sku: 'x' } }; + const original = { query: { sku: 'x' } }; + const response = { sku: 'x', count: 7 }; + let complete; + const tools = { read_inventory: { + effect_class: 'SE0', determinism: 'deterministic', + impl(input) { + if (mutateInImplementation) input.query.sku = 'changed'; + return new Promise(resolve => { complete = resolve; }); + } + } }; + const recorder = replay.createReplayer(tools, { mode: 'record', store }); + const pending = recorder.call('read_inventory', args); + if (!mutateInImplementation) args.query.sku = 'changed'; + complete(response); + assert.deepStrictEqual(await pending, response); + assert.deepStrictEqual(store.get('read_inventory', original).response, response); + assert.strictEqual(store.has('read_inventory', args), false, 'Mutation must not redirect fixture identity'); + assert.strictEqual(recorder.calls[0].fixture_key, store.key('read_inventory', original)); + const replayer = replay.createReplayer(tools, { mode: 'replay', store }); + assert.deepStrictEqual(replayer.call('read_inventory', original), response); + assert.throws(() => replayer.call('read_inventory', args), error => error.code === 'tool.fixture_missing'); + } finally { cleanup(dir); } + }); + } + + await asyncTest('synchronous implementations retain an immediate return and invocation arguments', () => { + const dir = tempDir('sync-arguments'); + try { + const store = new replay.FixtureStore(dir); + const args = { query: { sku: 'x' } }; + const response = { sku: 'x', count: 7 }; + const tools = { read_inventory: { + effect_class: 'SE0', determinism: 'deterministic', + impl(input) { input.query.sku = 'changed'; return response; } + } }; + const recorder = replay.createReplayer(tools, { mode: 'record', store }); + assert.strictEqual(recorder.call('read_inventory', args), response); + assert.deepStrictEqual(store.get('read_inventory', { query: { sku: 'x' } }).response, response); + assert.strictEqual(store.has('read_inventory', args), false); + } finally { cleanup(dir); } + }); + + await asyncTest('effect checks still refuse an async implementation before calling it', () => { + const dir = tempDir('async-effects'); + try { + const store = new replay.FixtureStore(dir); + let liveCalls = 0; + const tools = { charge_card: { + effect_class: 'SE4', determinism: 'nondeterministic', + async impl() { liveCalls += 1; return { charged: true }; } + } }; + const recorder = replay.createReplayer(tools, { mode: 'record', store, maxEffectClass: 'SE2' }); + assert.throws(() => recorder.call('charge_card', {}), error => error.code === 'tool.effect_forbidden'); + const replayer = replay.createReplayer(tools, { mode: 'replay', store, maxEffectClass: 'SE4' }); + assert.throws(() => replayer.call('charge_card', {}), error => error.code === 'tool.effect_forbidden'); + assert.strictEqual(liveCalls, 0); + assert.deepStrictEqual(fs.readdirSync(dir), []); + } finally { cleanup(dir); } + }); + + finish('replay async'); +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/tests/scripts/eval-harness-package.test.js b/tests/scripts/eval-harness-package.test.js index 5d413ca8c..39faa1797 100644 --- a/tests/scripts/eval-harness-package.test.js +++ b/tests/scripts/eval-harness-package.test.js @@ -111,10 +111,28 @@ try { test('failed checks remain visible to aggregation and return a failing exit', () => { const helper = path.join(repo, 'tests/lib/eval-harness/helpers.js'); - const script = `const h=require(${JSON.stringify(helper)});h.test('pass fixture',()=>{});h.test('failure fixture',()=>{throw new Error('synthetic failure');});h.finish('count fixture');`; + const script = `const h=require(${JSON.stringify(helper)});(async()=>{ + h.test('pass fixture',()=>{}); + h.test('failure fixture',()=>{throw new Error('synthetic failure');}); + await h.asyncTest('async pass fixture',async()=>{}); + await h.asyncTest('async failure fixture',async()=>{await Promise.resolve();throw new Error('synthetic async failure');}); + h.finish('count fixture'); + })().catch(()=>process.exit(1));`; const result = command(process.execPath, ['-e', script], { cwd: work }); assert.strictEqual(result.status, 1); - assert.deepStrictEqual(counts(result.stdout), { passed: 1, failed: 1 }); + assert.deepStrictEqual(counts(result.stdout), { passed: 2, failed: 2 }); + }); + + test('an unsettled async check times out instead of exiting successfully without a summary', () => { + const helper = path.join(repo, 'tests/lib/eval-harness/helpers.js'); + const script = `const h=require(${JSON.stringify(helper)});(async()=>{ + await h.asyncTest('pending fixture',()=>new Promise(()=>{}),25); + h.finish('pending fixture'); + })().catch(()=>process.exit(1));`; + const result = command(process.execPath, ['-e', script], { cwd: work }); + assert.strictEqual(result.status, 1); + assert.deepStrictEqual(counts(result.stdout), { passed: 0, failed: 1 }); + assert.match(result.stdout, /timed out after 25ms/); }); } finally { cleanup(work);