mirror of
https://github.com/affaan-m/ECC.git
synced 2026-11-17 13:32:18 +00:00
CLASSIFIER_NOTICE matched by prefix, so a pasted notice followed by a user question was dropped from both summary paths. Anchor it to the whole message so trailing user text is kept, and add a regression test. Also pass ECC_SKIP_LLM_SUMMARY=1 in the withTranscript fixture helper so an inherited ECC_LLM_SUMMARY_INTERVAL=1 cannot trigger claude -p. Refs #3867 Assisted-by: Claude <noreply@anthropic.com>
347 lines
14 KiB
JavaScript
347 lines
14 KiB
JavaScript
'use strict';
|
|
/**
|
|
* Tests for scripts/lib/llm-summary.js
|
|
*
|
|
* Run with: node tests/lib/llm-summary.test.js
|
|
*/
|
|
|
|
const assert = require('assert');
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
const path = require('path');
|
|
|
|
const { extractConversationText, isClassifierNotice, getContextRemainingPct, getContextThreshold, getLLMModel, generateSessionSummary } = require('../../scripts/lib/llm-summary');
|
|
|
|
console.log('=== Testing llm-summary.js ===\n');
|
|
|
|
let passed = 0;
|
|
let failed = 0;
|
|
|
|
function test(desc, fn) {
|
|
try {
|
|
fn();
|
|
console.log(` ✓ ${desc}`);
|
|
passed++;
|
|
} catch (e) {
|
|
console.log(` ✗ ${desc}: ${e.message}`);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
let seq = 0;
|
|
const transcriptDir = fs.mkdtempSync(path.join(os.tmpdir(), 'llm-summary-test-'));
|
|
function writeTranscript(lines) {
|
|
seq++;
|
|
const p = path.join(transcriptDir, `transcript-${seq}.jsonl`);
|
|
fs.writeFileSync(p, lines.join('\n') + '\n');
|
|
return p;
|
|
}
|
|
|
|
function userEntry(text) {
|
|
return JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text }] } });
|
|
}
|
|
|
|
function assistantEntry(text) {
|
|
return JSON.stringify({
|
|
type: 'assistant',
|
|
message: {
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text }],
|
|
usage: { input_tokens: 1000, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- getLLMModel ---
|
|
console.log('getLLMModel:');
|
|
|
|
test('returns haiku by default', () => {
|
|
const orig = process.env.ECC_LLM_SUMMARY_MODEL;
|
|
delete process.env.ECC_LLM_SUMMARY_MODEL;
|
|
assert.strictEqual(getLLMModel(), 'haiku');
|
|
if (orig !== undefined) process.env.ECC_LLM_SUMMARY_MODEL = orig;
|
|
});
|
|
|
|
test('reads ECC_LLM_SUMMARY_MODEL env var', () => {
|
|
const orig = process.env.ECC_LLM_SUMMARY_MODEL;
|
|
process.env.ECC_LLM_SUMMARY_MODEL = 'sonnet';
|
|
assert.strictEqual(getLLMModel(), 'sonnet');
|
|
if (orig !== undefined) process.env.ECC_LLM_SUMMARY_MODEL = orig;
|
|
else delete process.env.ECC_LLM_SUMMARY_MODEL;
|
|
});
|
|
|
|
// --- getContextThreshold ---
|
|
console.log('\ngetContextThreshold:');
|
|
|
|
test('returns 20 by default', () => {
|
|
const orig = process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
delete process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
assert.strictEqual(getContextThreshold(), 20);
|
|
if (orig !== undefined) process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD = orig;
|
|
});
|
|
|
|
test('reads ECC_LLM_SUMMARY_CONTEXT_THRESHOLD env var', () => {
|
|
const orig = process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD = '70';
|
|
assert.strictEqual(getContextThreshold(), 70);
|
|
if (orig !== undefined) process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD = orig;
|
|
else delete process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
});
|
|
|
|
test('falls back to 20 on invalid value', () => {
|
|
const orig = process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD = 'notanumber';
|
|
assert.strictEqual(getContextThreshold(), 20);
|
|
if (orig !== undefined) process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD = orig;
|
|
else delete process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
});
|
|
|
|
test('falls back to 20 when value exceeds 100', () => {
|
|
const orig = process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD = '150';
|
|
assert.strictEqual(getContextThreshold(), 20);
|
|
if (orig !== undefined) process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD = orig;
|
|
else delete process.env.ECC_LLM_SUMMARY_CONTEXT_THRESHOLD;
|
|
});
|
|
|
|
// --- extractConversationText ---
|
|
console.log('\nextractConversationText:');
|
|
|
|
test('returns null for missing file', () => {
|
|
assert.strictEqual(extractConversationText('/nonexistent/path.jsonl'), null);
|
|
});
|
|
|
|
test('returns null for empty transcript', () => {
|
|
const p = writeTranscript([]);
|
|
assert.strictEqual(extractConversationText(p), null);
|
|
});
|
|
|
|
test('extracts user and assistant turns', () => {
|
|
const p = writeTranscript([userEntry('Hello, can you help?'), assistantEntry('Sure, what do you need?')]);
|
|
const result = extractConversationText(p);
|
|
assert.ok(result.includes('User:'));
|
|
assert.ok(result.includes('Claude:'));
|
|
assert.ok(result.includes('Hello, can you help?'));
|
|
});
|
|
|
|
test('truncates user text to 400 chars', () => {
|
|
const p = writeTranscript([userEntry('x'.repeat(500))]);
|
|
const result = extractConversationText(p);
|
|
assert.ok(result !== null);
|
|
assert.ok(!result.includes('x'.repeat(401)));
|
|
});
|
|
|
|
test('skips unparseable lines gracefully', () => {
|
|
const p = writeTranscript(['not valid json', userEntry('valid message')]);
|
|
const result = extractConversationText(p);
|
|
assert.ok(result !== null);
|
|
assert.ok(result.includes('valid message'));
|
|
});
|
|
|
|
test('limits to last 25 turns', () => {
|
|
const lines = [];
|
|
for (let i = 0; i < 30; i++) lines.push(userEntry(`message ${i}`));
|
|
const p = writeTranscript(lines);
|
|
const result = extractConversationText(p);
|
|
assert.ok(result.includes('message 29'));
|
|
assert.ok(!result.includes('message 4'));
|
|
});
|
|
|
|
test('collapses newlines to spaces', () => {
|
|
const p = writeTranscript([userEntry('line one\nline two')]);
|
|
const result = extractConversationText(p);
|
|
assert.ok(!result.includes('\nline two'));
|
|
assert.ok(result.includes('line one line two'));
|
|
});
|
|
|
|
// Regression (#3867): the harness notice after a safety-classifier stop is
|
|
// recorded as a user entry; it must not reach the LLM summary input.
|
|
const CLASSIFIER_NOTICE_TEXT = 'Your response above was stopped by a safety classifier — this is not a tool or API error.';
|
|
|
|
test('drops safety-classifier notices from the LLM input', () => {
|
|
const p = writeTranscript([userEntry('Refactor the config loader'), userEntry(CLASSIFIER_NOTICE_TEXT), assistantEntry('Done.')]);
|
|
const result = extractConversationText(p);
|
|
assert.ok(result.includes('Refactor the config loader'));
|
|
assert.ok(!result.includes('safety classifier'), `classifier notice must not reach the LLM:\n${result}`);
|
|
});
|
|
|
|
test('keeps user text pasted after the notice', () => {
|
|
const text = `${CLASSIFIER_NOTICE_TEXT} Why did this happen?`;
|
|
assert.ok(extractConversationText(writeTranscript([userEntry(text)])).includes('Why did this happen?'));
|
|
});
|
|
|
|
test('returns null when the only user turn is a classifier notice', () => {
|
|
const p = writeTranscript([userEntry(CLASSIFIER_NOTICE_TEXT)]);
|
|
assert.strictEqual(extractConversationText(p), null);
|
|
});
|
|
|
|
test('isClassifierNotice matches only the whole notice', () => {
|
|
assert.strictEqual(isClassifierNotice(CLASSIFIER_NOTICE_TEXT), true);
|
|
assert.strictEqual(isClassifierNotice(`${CLASSIFIER_NOTICE_TEXT} Why did this happen?`), false);
|
|
assert.strictEqual(isClassifierNotice('Why did I see "Your response above was stopped by a safety classifier"?'), false);
|
|
});
|
|
|
|
// --- getContextRemainingPct ---
|
|
console.log('\ngetContextRemainingPct:');
|
|
|
|
test('returns null for missing file', () => {
|
|
assert.strictEqual(getContextRemainingPct('/nonexistent.jsonl'), null);
|
|
});
|
|
|
|
test('returns null for transcript with no usage data', () => {
|
|
const p = writeTranscript([userEntry('hi')]);
|
|
assert.strictEqual(getContextRemainingPct(p), null);
|
|
});
|
|
|
|
test('returns numeric percentage for transcript with usage data', () => {
|
|
const parsed = JSON.parse(assistantEntry('ok'));
|
|
const entry = { ...parsed, message: { ...parsed.message, model: 'claude-opus-4-5[1m]' } };
|
|
const p = writeTranscript([JSON.stringify(entry)]);
|
|
const pct = getContextRemainingPct(p);
|
|
assert.ok(typeof pct === 'number');
|
|
assert.ok(pct >= 0 && pct <= 100);
|
|
});
|
|
|
|
for (const [label, model, nativeBudget, explicitWindow, expected, tokens = 330000] of [
|
|
['inferred window uses native compaction budget conservatively', 'unknown-model', '400000', '', 18],
|
|
['unknown capacity without a budget stays unknown', 'unknown-model', '', '', null],
|
|
['summary precedes native compaction even with confirmed model capacity', 'claude-opus-4-5[1m]', '400000', '', 18],
|
|
['explicit ECC window remains authoritative', 'claude-opus-4-5[1m]', '786432', '400000', 18],
|
|
['invalid compaction budget does not certify an inferred window', 'unknown-model', 'invalid', '', null],
|
|
['native budget cannot exceed confirmed model capacity', 'claude-opus-4-5[1m]', '2000000', '', 67],
|
|
['explicit capacity does not defer a smaller native compaction budget', 'claude-opus-4-5[1m]', '400000', '1000000', 18],
|
|
['native 786k compaction receives a timely summary on a 1M model', 'claude-opus-4-5[1m]', '786432', '', 10, 710000],
|
|
['confirmed capacity remains usable without a native budget', 'claude-opus-4-5[1m]', '', '', 67],
|
|
['known model family also respects the earlier native budget', 'claude-fable-5', '400000', '', 18],
|
|
['unknown capacity rejects a numeric native prefix', 'unknown-model', '200000tokens', '', null, 180000],
|
|
['unknown capacity rejects a fractional native budget', 'unknown-model', '200000.5', '', null, 180000],
|
|
['unknown capacity rejects an unsafe native integer', 'unknown-model', '9007199254740992', '', null, 180000],
|
|
['invalid explicit prefix does not override a valid native budget', 'unknown-model', '400000', '200000tokens', 55, 180000],
|
|
]) {
|
|
test(label, () => {
|
|
const keys = ['ECC_CONTEXT_WINDOW_TOKENS', 'CLAUDE_CODE_AUTO_COMPACT_WINDOW'];
|
|
const previous = keys.map(key => process.env[key]);
|
|
try {
|
|
process.env.ECC_CONTEXT_WINDOW_TOKENS = explicitWindow;
|
|
process.env.CLAUDE_CODE_AUTO_COMPACT_WINDOW = nativeBudget;
|
|
const parsed = JSON.parse(assistantEntry('preserve context'));
|
|
const entry = {
|
|
...parsed,
|
|
message: { ...parsed.message, model, usage: { ...parsed.message.usage, input_tokens: tokens } },
|
|
};
|
|
assert.strictEqual(getContextRemainingPct(writeTranscript([JSON.stringify(entry)])), expected);
|
|
if (model.includes('[1m]') || model === 'claude-fable-5') {
|
|
const { resolveContextWindowTokens } = require('../../scripts/lib/transcript-context');
|
|
assert.strictEqual(resolveContextWindowTokens(tokens, model), explicitWindow ? Number(explicitWindow) : 1000000,
|
|
'summary scheduling must not change model capacity resolution');
|
|
}
|
|
} finally {
|
|
keys.forEach((key, index) => {
|
|
if (previous[index] === undefined) delete process.env[key];
|
|
else process.env[key] = previous[index];
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
// --- generateSessionSummary ---
|
|
console.log('\ngenerateSessionSummary:');
|
|
|
|
test('returns null when ECC_SKIP_LLM_SUMMARY is set', () => {
|
|
const orig = process.env.ECC_SKIP_LLM_SUMMARY;
|
|
process.env.ECC_SKIP_LLM_SUMMARY = '1';
|
|
const p = writeTranscript([userEntry('test')]);
|
|
assert.strictEqual(generateSessionSummary(p), null);
|
|
if (orig !== undefined) process.env.ECC_SKIP_LLM_SUMMARY = orig;
|
|
else delete process.env.ECC_SKIP_LLM_SUMMARY;
|
|
});
|
|
|
|
test('reports the explicit recursion guard as a deliberate skip', () => {
|
|
const orig = process.env.ECC_SKIP_LLM_SUMMARY;
|
|
process.env.ECC_SKIP_LLM_SUMMARY = '1';
|
|
try {
|
|
const skipped = [];
|
|
assert.strictEqual(generateSessionSummary('/nonexistent.jsonl', {
|
|
onSkip: reason => skipped.push(reason)
|
|
}), null);
|
|
assert.deepStrictEqual(skipped, ['ECC_SKIP_LLM_SUMMARY']);
|
|
} finally {
|
|
if (orig !== undefined) process.env.ECC_SKIP_LLM_SUMMARY = orig;
|
|
else delete process.env.ECC_SKIP_LLM_SUMMARY;
|
|
}
|
|
});
|
|
|
|
test('returns null for missing transcript (no conversation to summarize)', () => {
|
|
const orig = process.env.ECC_SKIP_LLM_SUMMARY;
|
|
delete process.env.ECC_SKIP_LLM_SUMMARY;
|
|
assert.strictEqual(generateSessionSummary('/nonexistent.jsonl'), null);
|
|
if (orig !== undefined) process.env.ECC_SKIP_LLM_SUMMARY = orig;
|
|
});
|
|
|
|
test('marks the spawned summarizer so its Stop hook cannot create resume state', () => {
|
|
const source = fs.readFileSync(
|
|
path.join(__dirname, '..', '..', 'scripts', 'lib', 'llm-summary.js'),
|
|
'utf8'
|
|
);
|
|
assert.match(source, /ECC_LLM_SUMMARY_SUBPROCESS:\s*'1'/);
|
|
});
|
|
|
|
// --- Results ---
|
|
// Execute the real helper with a controlled clock and child-process boundary.
|
|
// These cases never invoke Claude or use the developer's authentication.
|
|
function summaryProbe(deadline, now = 100000, spawnResult = { status: 0, stdout: 'summary' }) {
|
|
const vm = require('vm');
|
|
const source = fs.readFileSync(path.join(__dirname, '../../scripts/lib/llm-summary.js'), 'utf8');
|
|
const calls = [];
|
|
const env = deadline === undefined ? {} : { ECC_HOOK_DEADLINE_MS: String(deadline) };
|
|
const sandbox = {
|
|
module: { exports: {} },
|
|
process: { env, platform: process.platform },
|
|
Date: { now: () => now },
|
|
require: name => name === 'child_process'
|
|
? { spawnSync: (...args) => { calls.push(args); return spawnResult; } }
|
|
: require(name)
|
|
};
|
|
vm.runInNewContext(source, sandbox, { filename: path.join(__dirname, '../../scripts/lib/llm-summary.js') });
|
|
const skipped = [];
|
|
const result = sandbox.module.exports.generateSessionSummary(
|
|
writeTranscript([userEntry('Preserve mechanical resume state')]),
|
|
{ onSkip: reason => skipped.push(reason) }
|
|
);
|
|
return { calls, skipped, result };
|
|
}
|
|
|
|
test('limits summarizer to remaining hook budget minus persistence reserve', () => {
|
|
const probe = summaryProbe(105000);
|
|
assert.strictEqual(probe.result, 'summary');
|
|
assert.strictEqual(probe.calls[0][2].timeout, 4000);
|
|
assert.strictEqual(probe.skipped.length, 0);
|
|
});
|
|
|
|
test('retains the standalone timeout and caps longer inherited budgets', () => {
|
|
assert.strictEqual(summaryProbe(undefined).calls[0][2].timeout, 90000);
|
|
assert.strictEqual(summaryProbe(300000).calls[0][2].timeout, 90000);
|
|
});
|
|
|
|
for (const deadline of [99999, 100000, 100999, 101000, '', -1, 'invalid', 'Infinity', '1.5', Number.MAX_SAFE_INTEGER + 1]) {
|
|
test(`skips summarizer with exhausted or invalid deadline ${deadline}`, () => {
|
|
const probe = summaryProbe(deadline);
|
|
assert.strictEqual(probe.result, null);
|
|
assert.strictEqual(probe.calls.length, 0, 'No child should start without a safe budget');
|
|
assert.strictEqual(probe.skipped.length, 1, 'Skip must be distinguishable from Claude failure');
|
|
});
|
|
}
|
|
|
|
test('keeps process failures distinct from deliberate budget skips', () => {
|
|
const probe = summaryProbe(105000, 100000, { status: 1, stdout: '' });
|
|
assert.strictEqual(probe.result, null);
|
|
assert.strictEqual(probe.calls.length, 1);
|
|
assert.strictEqual(probe.skipped.length, 0);
|
|
});
|
|
|
|
console.log('\n=== Test Results ===');
|
|
console.log(`Passed: ${passed}`);
|
|
console.log(`Failed: ${failed}`);
|
|
console.log(`Total: ${passed + failed}`);
|
|
process.exit(failed > 0 ? 1 : 0);
|