mirror of
https://github.com/affaan-m/ECC.git
synced 2026-11-17 05:22:17 +00:00
fix(eval): record fulfilled async tool responses
This commit is contained in:
1 parent
d2ef268059
commit
8f0dd2ca3a
5 files changed
+239
-6
No files matched your search
@@ -313,6 +313,18 @@ an implementation. The legacy `EFFECT_FENCE_PRELOAD` export remains for import
|
||||
compatibility, but loading that file always throws `gate.isolation_required`.
|
||||
It no longer attempts JavaScript interception.
|
||||
|
||||
Record-mode calls preserve synchronous results. If an implementation returns a
|
||||
Promise or thenable, await `call()` before consuming the result: the fixture and
|
||||
`recorded` trace are written only after fulfillment. An implementation rejection
|
||||
propagates without writing a fixture or successful trace and leaves any previous
|
||||
fixture unchanged.
|
||||
Replay remains synchronous and never invokes the implementation.
|
||||
Fixture keys use a JSON snapshot of the arguments at invocation, before the
|
||||
implementation runs; later argument mutations cannot redirect the recording.
|
||||
Re-record fixtures previously captured from async implementations: an empty
|
||||
object recorded from an unresolved Promise still has a valid hash and cannot
|
||||
be distinguished automatically from a real empty response.
|
||||
|
||||
## Offline receipts
|
||||
|
||||
```sh
|
||||
|
||||
@@ -86,6 +86,7 @@ class FixtureStore {
|
||||
/**
|
||||
* tools: { name: { effect_class, determinism: 'deterministic'|'nondeterministic', impl(args) } }
|
||||
* options: { mode: 'record'|'replay', store: FixtureStore, maxEffectClass: 'SE2', onCall(entry) }
|
||||
* Record calls return a Promise only when the implementation returns a thenable.
|
||||
*/
|
||||
function createReplayer(tools, options = {}) {
|
||||
const mode = options.mode || 'replay';
|
||||
@@ -136,10 +137,20 @@ function createReplayer(tools, options = {}) {
|
||||
emit({ tool: name, mode, status: 'replayed', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash });
|
||||
return record.response;
|
||||
}
|
||||
// Keep fixture identity at invocation time, even if the caller or the
|
||||
// implementation changes a nested argument before an async result settles.
|
||||
const invocationArgs = JSON.parse(canonicalJson(args));
|
||||
const response = tool.impl(args);
|
||||
const record = store.put(name, args, response);
|
||||
emit({ tool: name, mode, status: 'recorded', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash });
|
||||
return response;
|
||||
const recordResponse = (resolved) => {
|
||||
const record = store.put(name, invocationArgs, resolved);
|
||||
emit({ tool: name, mode, status: 'recorded', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash });
|
||||
return resolved;
|
||||
};
|
||||
// Persist settled values, never the Promise object or a failed request.
|
||||
if (response && typeof response.then === 'function') {
|
||||
return Promise.resolve(response).then(recordResponse);
|
||||
}
|
||||
return recordResponse(response);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -20,6 +20,28 @@ function test(name, fn) {
|
||||
}
|
||||
}
|
||||
|
||||
async function asyncTest(name, fn, timeoutMs = 5000) {
|
||||
let timer;
|
||||
try {
|
||||
// Keep a referenced timer: an unresolved Promise alone lets Node exit 0
|
||||
// before finish() can report an incomplete test as a failure.
|
||||
await Promise.race([
|
||||
Promise.resolve().then(fn),
|
||||
new Promise((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs);
|
||||
})
|
||||
]);
|
||||
console.log(` ✓ ${name}`);
|
||||
passed += 1;
|
||||
} catch (error) {
|
||||
console.log(` ✗ ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
failed += 1;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function tempDir(prefix) {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), `ecc-eval-harness-${prefix}-`));
|
||||
}
|
||||
@@ -55,4 +77,4 @@ function runNpm(args, options = {}) {
|
||||
|
||||
const fixedClock = () => new Date('2026-09-02T00:00:00.000Z');
|
||||
|
||||
module.exports = { test, tempDir, cleanup, finish, fixedClock, runNpm };
|
||||
module.exports = { test, asyncTest, tempDir, cleanup, finish, fixedClock, runNpm };
|
||||
@@ -0,0 +1,170 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const replay = require('../../../scripts/lib/eval-harness/replay');
|
||||
const { asyncTest, tempDir, cleanup, finish } = require('./helpers');
|
||||
|
||||
async function main() {
|
||||
console.log('\n=== eval-harness async replay ===\n');
|
||||
|
||||
await asyncTest('record waits for an async tool before persisting its resolved response', async () => {
|
||||
const dir = tempDir('async-record');
|
||||
try {
|
||||
const store = new replay.FixtureStore(dir);
|
||||
const args = { sku: 'x' };
|
||||
const response = { sku: 'x', count: 7 };
|
||||
const trace = [];
|
||||
let complete;
|
||||
let liveCalls = 0;
|
||||
const tools = { read_inventory: {
|
||||
effect_class: 'SE0', determinism: 'deterministic',
|
||||
impl() { liveCalls += 1; return new Promise(resolve => { complete = resolve; }); }
|
||||
} };
|
||||
const recorder = replay.createReplayer(tools, { mode: 'record', store, onCall: entry => trace.push(entry) });
|
||||
const pending = recorder.call('read_inventory', args);
|
||||
|
||||
assert.strictEqual(store.has('read_inventory', args), false, 'Pending response is not a fixture');
|
||||
assert.deepStrictEqual(recorder.calls, [], 'Pending response is not a successful call');
|
||||
assert.deepStrictEqual(trace, []);
|
||||
complete(response);
|
||||
|
||||
assert.deepStrictEqual(await pending, response);
|
||||
assert.deepStrictEqual(store.get('read_inventory', args).response, response);
|
||||
assert.strictEqual(recorder.calls.length, 1);
|
||||
assert.strictEqual(recorder.calls[0].status, 'recorded');
|
||||
assert.deepStrictEqual(trace, recorder.calls);
|
||||
|
||||
const replayer = replay.createReplayer(tools, { mode: 'replay', store });
|
||||
assert.deepStrictEqual(replayer.call('read_inventory', args), response);
|
||||
assert.strictEqual(liveCalls, 1, 'Replay must not call the live async implementation');
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
await asyncTest('async rejection preserves the error without creating a successful fixture or trace', async () => {
|
||||
const dir = tempDir('async-rejection');
|
||||
try {
|
||||
const store = new replay.FixtureStore(dir);
|
||||
const failure = new Error('inventory request failed');
|
||||
const trace = [];
|
||||
const tools = { read_inventory: {
|
||||
effect_class: 'SE0', determinism: 'nondeterministic',
|
||||
async impl() { throw failure; }
|
||||
} };
|
||||
const recorder = replay.createReplayer(tools, { mode: 'record', store, onCall: entry => trace.push(entry) });
|
||||
await assert.rejects(recorder.call('read_inventory', {}), error => error === failure);
|
||||
assert.deepStrictEqual(fs.readdirSync(dir), []);
|
||||
assert.deepStrictEqual(recorder.calls, []);
|
||||
assert.deepStrictEqual(trace, []);
|
||||
const replayer = replay.createReplayer(tools, { mode: 'replay', store });
|
||||
assert.throws(() => replayer.call('read_inventory', {}), error => error.code === 'tool.fixture_missing');
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
await asyncTest('a failed async refresh leaves an existing fixture unchanged', async () => {
|
||||
const dir = tempDir('async-refresh');
|
||||
try {
|
||||
const store = new replay.FixtureStore(dir);
|
||||
const args = { sku: 'x' };
|
||||
store.put('read_inventory', args, { count: 7 });
|
||||
const before = fs.readFileSync(store.pathFor(store.key('read_inventory', args)));
|
||||
const tools = { read_inventory: {
|
||||
effect_class: 'SE0', determinism: 'nondeterministic',
|
||||
async impl() { throw new Error('offline'); }
|
||||
} };
|
||||
const recorder = replay.createReplayer(tools, { mode: 'record', store });
|
||||
await assert.rejects(recorder.call('read_inventory', args), /offline/);
|
||||
assert.deepStrictEqual(fs.readFileSync(store.pathFor(store.key('read_inventory', args))), before);
|
||||
assert.deepStrictEqual(recorder.calls, []);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
await asyncTest('record assimilates thenable tool results before hashing', async () => {
|
||||
const dir = tempDir('thenable');
|
||||
try {
|
||||
const store = new replay.FixtureStore(dir);
|
||||
const response = { count: 7 };
|
||||
const tools = { read_inventory: {
|
||||
effect_class: 'SE0', determinism: 'deterministic',
|
||||
impl() { return { then(resolve) { resolve(response); } }; }
|
||||
} };
|
||||
const recorder = replay.createReplayer(tools, { mode: 'record', store });
|
||||
assert.deepStrictEqual(await recorder.call('read_inventory', {}), response);
|
||||
assert.deepStrictEqual(store.get('read_inventory', {}).response, response);
|
||||
assert.strictEqual(recorder.calls[0].status, 'recorded');
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
for (const mutateInImplementation of [false, true]) {
|
||||
await asyncTest(`records invocation arguments despite ${mutateInImplementation ? 'implementation' : 'caller'} mutation`, async () => {
|
||||
const dir = tempDir('async-arguments');
|
||||
try {
|
||||
const store = new replay.FixtureStore(dir);
|
||||
const args = { query: { sku: 'x' } };
|
||||
const original = { query: { sku: 'x' } };
|
||||
const response = { sku: 'x', count: 7 };
|
||||
let complete;
|
||||
const tools = { read_inventory: {
|
||||
effect_class: 'SE0', determinism: 'deterministic',
|
||||
impl(input) {
|
||||
if (mutateInImplementation) input.query.sku = 'changed';
|
||||
return new Promise(resolve => { complete = resolve; });
|
||||
}
|
||||
} };
|
||||
const recorder = replay.createReplayer(tools, { mode: 'record', store });
|
||||
const pending = recorder.call('read_inventory', args);
|
||||
if (!mutateInImplementation) args.query.sku = 'changed';
|
||||
complete(response);
|
||||
assert.deepStrictEqual(await pending, response);
|
||||
assert.deepStrictEqual(store.get('read_inventory', original).response, response);
|
||||
assert.strictEqual(store.has('read_inventory', args), false, 'Mutation must not redirect fixture identity');
|
||||
assert.strictEqual(recorder.calls[0].fixture_key, store.key('read_inventory', original));
|
||||
const replayer = replay.createReplayer(tools, { mode: 'replay', store });
|
||||
assert.deepStrictEqual(replayer.call('read_inventory', original), response);
|
||||
assert.throws(() => replayer.call('read_inventory', args), error => error.code === 'tool.fixture_missing');
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
}
|
||||
|
||||
await asyncTest('synchronous implementations retain an immediate return and invocation arguments', () => {
|
||||
const dir = tempDir('sync-arguments');
|
||||
try {
|
||||
const store = new replay.FixtureStore(dir);
|
||||
const args = { query: { sku: 'x' } };
|
||||
const response = { sku: 'x', count: 7 };
|
||||
const tools = { read_inventory: {
|
||||
effect_class: 'SE0', determinism: 'deterministic',
|
||||
impl(input) { input.query.sku = 'changed'; return response; }
|
||||
} };
|
||||
const recorder = replay.createReplayer(tools, { mode: 'record', store });
|
||||
assert.strictEqual(recorder.call('read_inventory', args), response);
|
||||
assert.deepStrictEqual(store.get('read_inventory', { query: { sku: 'x' } }).response, response);
|
||||
assert.strictEqual(store.has('read_inventory', args), false);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
await asyncTest('effect checks still refuse an async implementation before calling it', () => {
|
||||
const dir = tempDir('async-effects');
|
||||
try {
|
||||
const store = new replay.FixtureStore(dir);
|
||||
let liveCalls = 0;
|
||||
const tools = { charge_card: {
|
||||
effect_class: 'SE4', determinism: 'nondeterministic',
|
||||
async impl() { liveCalls += 1; return { charged: true }; }
|
||||
} };
|
||||
const recorder = replay.createReplayer(tools, { mode: 'record', store, maxEffectClass: 'SE2' });
|
||||
assert.throws(() => recorder.call('charge_card', {}), error => error.code === 'tool.effect_forbidden');
|
||||
const replayer = replay.createReplayer(tools, { mode: 'replay', store, maxEffectClass: 'SE4' });
|
||||
assert.throws(() => replayer.call('charge_card', {}), error => error.code === 'tool.effect_forbidden');
|
||||
assert.strictEqual(liveCalls, 0);
|
||||
assert.deepStrictEqual(fs.readdirSync(dir), []);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
finish('replay async');
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -111,10 +111,28 @@ try {
|
||||
|
||||
test('failed checks remain visible to aggregation and return a failing exit', () => {
|
||||
const helper = path.join(repo, 'tests/lib/eval-harness/helpers.js');
|
||||
const script = `const h=require(${JSON.stringify(helper)});h.test('pass fixture',()=>{});h.test('failure fixture',()=>{throw new Error('synthetic failure');});h.finish('count fixture');`;
|
||||
const script = `const h=require(${JSON.stringify(helper)});(async()=>{
|
||||
h.test('pass fixture',()=>{});
|
||||
h.test('failure fixture',()=>{throw new Error('synthetic failure');});
|
||||
await h.asyncTest('async pass fixture',async()=>{});
|
||||
await h.asyncTest('async failure fixture',async()=>{await Promise.resolve();throw new Error('synthetic async failure');});
|
||||
h.finish('count fixture');
|
||||
})().catch(()=>process.exit(1));`;
|
||||
const result = command(process.execPath, ['-e', script], { cwd: work });
|
||||
assert.strictEqual(result.status, 1);
|
||||
assert.deepStrictEqual(counts(result.stdout), { passed: 1, failed: 1 });
|
||||
assert.deepStrictEqual(counts(result.stdout), { passed: 2, failed: 2 });
|
||||
});
|
||||
|
||||
test('an unsettled async check times out instead of exiting successfully without a summary', () => {
|
||||
const helper = path.join(repo, 'tests/lib/eval-harness/helpers.js');
|
||||
const script = `const h=require(${JSON.stringify(helper)});(async()=>{
|
||||
await h.asyncTest('pending fixture',()=>new Promise(()=>{}),25);
|
||||
h.finish('pending fixture');
|
||||
})().catch(()=>process.exit(1));`;
|
||||
const result = command(process.execPath, ['-e', script], { cwd: work });
|
||||
assert.strictEqual(result.status, 1);
|
||||
assert.deepStrictEqual(counts(result.stdout), { passed: 0, failed: 1 });
|
||||
assert.match(result.stdout, /timed out after 25ms/);
|
||||
});
|
||||
} finally {
|
||||
cleanup(work);
|
||||
|
||||
Reference in new issue
Block a user