diff --git a/scripts/run-evals-test.js b/scripts/run-evals-test.js index 381a8cf0..75fa70fb 100644 --- a/scripts/run-evals-test.js +++ b/scripts/run-evals-test.js @@ -90,6 +90,25 @@ test('rejects grader results that omit expectations', () => { assert.equal(parseGrading(raw, 2), null); }); +test('rejects null expectation entries without throwing', () => { + for (const expectations of [ + [null], + [{ text: 'first expectation', passed: true, evidence: 'observed' }, null], + ]) { + const raw = JSON.stringify({ + expectations, + summary: { + passed: expectations.length - 1, + failed: 1, + total: expectations.length, + pass_rate: (expectations.length - 1) / expectations.length, + }, + }); + + assert.equal(parseGrading(raw, expectations.length), null); + } +}); + test('rejects incomplete or inconsistent grader summaries', () => { const expectation = { text: 'expected behavior', passed: false, evidence: 'not observed' }; const cases = [ diff --git a/scripts/run-evals.js b/scripts/run-evals.js index 80138873..0d543b39 100644 --- a/scripts/run-evals.js +++ b/scripts/run-evals.js @@ -439,12 +439,13 @@ function parseGrading(raw, expectedCount) { const expectations = g.expectations; const summary = g.summary; const passed = Array.isArray(expectations) - ? expectations.filter((expectation) => expectation.passed === true).length + ? expectations.filter((expectation) => expectation?.passed === true).length : 0; const ok = Number.isInteger(expectedCount) && expectedCount > 0 && Array.isArray(expectations) && expectations.length === expectedCount && expectations.every((expectation) => + expectation !== null && typeof expectation.text === 'string' && typeof expectation.passed === 'boolean' && typeof expectation.evidence === 'string') &&