Merge remote-tracking branch 'origin/pr-3933' into stewardship/20261116

This commit is contained in:
haelyra committed 2026-11-16 13:22:31 -05:00
commit 8d78e08609
5 files changed
+239 -6

No files matched your search

@@ -313,6 +313,18 @@ an implementation. The legacy `EFFECT_FENCE_PRELOAD` export remains for import
compatibility, but loading that file always throws `gate.isolation_required`.
It no longer attempts JavaScript interception.
Record-mode calls preserve synchronous results. If an implementation returns a
Promise or thenable, await `call()` before consuming the result: the fixture and
`recorded` trace are written only after fulfillment. An implementation rejection
propagates without writing a fixture or successful trace and leaves any previous
fixture unchanged.
Replay remains synchronous and never invokes the implementation.
Fixture keys use a JSON snapshot of the arguments at invocation, before the
implementation runs; later argument mutations cannot redirect the recording.
Re-record fixtures previously captured from async implementations: an empty
object recorded from an unresolved Promise still has a valid hash and cannot
be distinguished automatically from a real empty response.
## Offline receipts
```sh
+14 -3
View File
@@ -86,6 +86,7 @@ class FixtureStore {
/**
* tools: { name: { effect_class, determinism: 'deterministic'|'nondeterministic', impl(args) } }
* options: { mode: 'record'|'replay', store: FixtureStore, maxEffectClass: 'SE2', onCall(entry) }
* Record calls return a Promise only when the implementation returns a thenable.
*/
function createReplayer(tools, options = {}) {
const mode = options.mode || 'replay';
@@ -136,10 +137,20 @@ function createReplayer(tools, options = {}) {
emit({ tool: name, mode, status: 'replayed', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash });
return record.response;
}
// Keep fixture identity at invocation time, even if the caller or the
// implementation changes a nested argument before an async result settles.
const invocationArgs = JSON.parse(canonicalJson(args));
const response = tool.impl(args);
const record = store.put(name, args, response);
emit({ tool: name, mode, status: 'recorded', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash });
return response;
const recordResponse = (resolved) => {
const record = store.put(name, invocationArgs, resolved);
emit({ tool: name, mode, status: 'recorded', fixture_key: record.key, args_hash: record.args_hash, response_hash: record.response_hash });
return resolved;
};
// Persist settled values, never the Promise object or a failed request.
if (response && typeof response.then === 'function') {
return Promise.resolve(response).then(recordResponse);
}
return recordResponse(response);
},
};
}
+23 -1
View File
@@ -20,6 +20,28 @@ function test(name, fn) {
}
}
async function asyncTest(name, fn, timeoutMs = 5000) {
let timer;
try {
// Keep a referenced timer: an unresolved Promise alone lets Node exit 0
// before finish() can report an incomplete test as a failure.
await Promise.race([
Promise.resolve().then(fn),
new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs);
})
]);
console.log(` ✓ ${name}`);
passed += 1;
} catch (error) {
console.log(` ✗ ${name}`);
console.log(` Error: ${error.message}`);
failed += 1;
} finally {
clearTimeout(timer);
}
}
function tempDir(prefix) {
return fs.mkdtempSync(path.join(os.tmpdir(), `ecc-eval-harness-${prefix}-`));
}
@@ -55,4 +77,4 @@ function runNpm(args, options = {}) {
const fixedClock = () => new Date('2026-09-02T00:00:00.000Z');
module.exports = { test, tempDir, cleanup, finish, fixedClock, runNpm };
module.exports = { test, asyncTest, tempDir, cleanup, finish, fixedClock, runNpm };
+170
View File
@@ -0,0 +1,170 @@
'use strict';
const assert = require('assert');
const fs = require('fs');
const replay = require('../../../scripts/lib/eval-harness/replay');
const { asyncTest, tempDir, cleanup, finish } = require('./helpers');
async function main() {
console.log('\n=== eval-harness async replay ===\n');
await asyncTest('record waits for an async tool before persisting its resolved response', async () => {
const dir = tempDir('async-record');
try {
const store = new replay.FixtureStore(dir);
const args = { sku: 'x' };
const response = { sku: 'x', count: 7 };
const trace = [];
let complete;
let liveCalls = 0;
const tools = { read_inventory: {
effect_class: 'SE0', determinism: 'deterministic',
impl() { liveCalls += 1; return new Promise(resolve => { complete = resolve; }); }
} };
const recorder = replay.createReplayer(tools, { mode: 'record', store, onCall: entry => trace.push(entry) });
const pending = recorder.call('read_inventory', args);
assert.strictEqual(store.has('read_inventory', args), false, 'Pending response is not a fixture');
assert.deepStrictEqual(recorder.calls, [], 'Pending response is not a successful call');
assert.deepStrictEqual(trace, []);
complete(response);
assert.deepStrictEqual(await pending, response);
assert.deepStrictEqual(store.get('read_inventory', args).response, response);
assert.strictEqual(recorder.calls.length, 1);
assert.strictEqual(recorder.calls[0].status, 'recorded');
assert.deepStrictEqual(trace, recorder.calls);
const replayer = replay.createReplayer(tools, { mode: 'replay', store });
assert.deepStrictEqual(replayer.call('read_inventory', args), response);
assert.strictEqual(liveCalls, 1, 'Replay must not call the live async implementation');
} finally { cleanup(dir); }
});
await asyncTest('async rejection preserves the error without creating a successful fixture or trace', async () => {
const dir = tempDir('async-rejection');
try {
const store = new replay.FixtureStore(dir);
const failure = new Error('inventory request failed');
const trace = [];
const tools = { read_inventory: {
effect_class: 'SE0', determinism: 'nondeterministic',
async impl() { throw failure; }
} };
const recorder = replay.createReplayer(tools, { mode: 'record', store, onCall: entry => trace.push(entry) });
await assert.rejects(recorder.call('read_inventory', {}), error => error === failure);
assert.deepStrictEqual(fs.readdirSync(dir), []);
assert.deepStrictEqual(recorder.calls, []);
assert.deepStrictEqual(trace, []);
const replayer = replay.createReplayer(tools, { mode: 'replay', store });
assert.throws(() => replayer.call('read_inventory', {}), error => error.code === 'tool.fixture_missing');
} finally { cleanup(dir); }
});
await asyncTest('a failed async refresh leaves an existing fixture unchanged', async () => {
const dir = tempDir('async-refresh');
try {
const store = new replay.FixtureStore(dir);
const args = { sku: 'x' };
store.put('read_inventory', args, { count: 7 });
const before = fs.readFileSync(store.pathFor(store.key('read_inventory', args)));
const tools = { read_inventory: {
effect_class: 'SE0', determinism: 'nondeterministic',
async impl() { throw new Error('offline'); }
} };
const recorder = replay.createReplayer(tools, { mode: 'record', store });
await assert.rejects(recorder.call('read_inventory', args), /offline/);
assert.deepStrictEqual(fs.readFileSync(store.pathFor(store.key('read_inventory', args))), before);
assert.deepStrictEqual(recorder.calls, []);
} finally { cleanup(dir); }
});
await asyncTest('record assimilates thenable tool results before hashing', async () => {
const dir = tempDir('thenable');
try {
const store = new replay.FixtureStore(dir);
const response = { count: 7 };
const tools = { read_inventory: {
effect_class: 'SE0', determinism: 'deterministic',
impl() { return { then(resolve) { resolve(response); } }; }
} };
const recorder = replay.createReplayer(tools, { mode: 'record', store });
assert.deepStrictEqual(await recorder.call('read_inventory', {}), response);
assert.deepStrictEqual(store.get('read_inventory', {}).response, response);
assert.strictEqual(recorder.calls[0].status, 'recorded');
} finally { cleanup(dir); }
});
for (const mutateInImplementation of [false, true]) {
await asyncTest(`records invocation arguments despite ${mutateInImplementation ? 'implementation' : 'caller'} mutation`, async () => {
const dir = tempDir('async-arguments');
try {
const store = new replay.FixtureStore(dir);
const args = { query: { sku: 'x' } };
const original = { query: { sku: 'x' } };
const response = { sku: 'x', count: 7 };
let complete;
const tools = { read_inventory: {
effect_class: 'SE0', determinism: 'deterministic',
impl(input) {
if (mutateInImplementation) input.query.sku = 'changed';
return new Promise(resolve => { complete = resolve; });
}
} };
const recorder = replay.createReplayer(tools, { mode: 'record', store });
const pending = recorder.call('read_inventory', args);
if (!mutateInImplementation) args.query.sku = 'changed';
complete(response);
assert.deepStrictEqual(await pending, response);
assert.deepStrictEqual(store.get('read_inventory', original).response, response);
assert.strictEqual(store.has('read_inventory', args), false, 'Mutation must not redirect fixture identity');
assert.strictEqual(recorder.calls[0].fixture_key, store.key('read_inventory', original));
const replayer = replay.createReplayer(tools, { mode: 'replay', store });
assert.deepStrictEqual(replayer.call('read_inventory', original), response);
assert.throws(() => replayer.call('read_inventory', args), error => error.code === 'tool.fixture_missing');
} finally { cleanup(dir); }
});
}
await asyncTest('synchronous implementations retain an immediate return and invocation arguments', () => {
const dir = tempDir('sync-arguments');
try {
const store = new replay.FixtureStore(dir);
const args = { query: { sku: 'x' } };
const response = { sku: 'x', count: 7 };
const tools = { read_inventory: {
effect_class: 'SE0', determinism: 'deterministic',
impl(input) { input.query.sku = 'changed'; return response; }
} };
const recorder = replay.createReplayer(tools, { mode: 'record', store });
assert.strictEqual(recorder.call('read_inventory', args), response);
assert.deepStrictEqual(store.get('read_inventory', { query: { sku: 'x' } }).response, response);
assert.strictEqual(store.has('read_inventory', args), false);
} finally { cleanup(dir); }
});
await asyncTest('effect checks still refuse an async implementation before calling it', () => {
const dir = tempDir('async-effects');
try {
const store = new replay.FixtureStore(dir);
let liveCalls = 0;
const tools = { charge_card: {
effect_class: 'SE4', determinism: 'nondeterministic',
async impl() { liveCalls += 1; return { charged: true }; }
} };
const recorder = replay.createReplayer(tools, { mode: 'record', store, maxEffectClass: 'SE2' });
assert.throws(() => recorder.call('charge_card', {}), error => error.code === 'tool.effect_forbidden');
const replayer = replay.createReplayer(tools, { mode: 'replay', store, maxEffectClass: 'SE4' });
assert.throws(() => replayer.call('charge_card', {}), error => error.code === 'tool.effect_forbidden');
assert.strictEqual(liveCalls, 0);
assert.deepStrictEqual(fs.readdirSync(dir), []);
} finally { cleanup(dir); }
});
finish('replay async');
}
main().catch(error => {
console.error(error);
process.exit(1);
});
+20 -2
View File
@@ -111,10 +111,28 @@ try {
test('failed checks remain visible to aggregation and return a failing exit', () => {
const helper = path.join(repo, 'tests/lib/eval-harness/helpers.js');
const script = `const h=require(${JSON.stringify(helper)});h.test('pass fixture',()=>{});h.test('failure fixture',()=>{throw new Error('synthetic failure');});h.finish('count fixture');`;
const script = `const h=require(${JSON.stringify(helper)});(async()=>{
h.test('pass fixture',()=>{});
h.test('failure fixture',()=>{throw new Error('synthetic failure');});
await h.asyncTest('async pass fixture',async()=>{});
await h.asyncTest('async failure fixture',async()=>{await Promise.resolve();throw new Error('synthetic async failure');});
h.finish('count fixture');
})().catch(()=>process.exit(1));`;
const result = command(process.execPath, ['-e', script], { cwd: work });
assert.strictEqual(result.status, 1);
assert.deepStrictEqual(counts(result.stdout), { passed: 1, failed: 1 });
assert.deepStrictEqual(counts(result.stdout), { passed: 2, failed: 2 });
});
test('an unsettled async check times out instead of exiting successfully without a summary', () => {
const helper = path.join(repo, 'tests/lib/eval-harness/helpers.js');
const script = `const h=require(${JSON.stringify(helper)});(async()=>{
await h.asyncTest('pending fixture',()=>new Promise(()=>{}),25);
h.finish('pending fixture');
})().catch(()=>process.exit(1));`;
const result = command(process.execPath, ['-e', script], { cwd: work });
assert.strictEqual(result.status, 1);
assert.deepStrictEqual(counts(result.stdout), { passed: 0, failed: 1 });
assert.match(result.stdout, /timed out after 25ms/);
});
} finally {
cleanup(work);