mirror of
https://github.com/affaan-m/ECC.git
synced 2026-11-17 05:22:17 +00:00
fix(hooks): drop classifier notices from LLM summary input
The safety-classifier notice filter only covered the mechanical summary. When generateSessionSummary runs (low context or message interval), extractConversationText read the raw transcript and passed the notice to the LLM as a user turn, so it could still reach saved context. Move the check into one shared isClassifierNotice() in scripts/lib/llm-summary.js and use it in both userMessageText() and extractConversationText(). Add LLM-path regression tests, and split the session-end classifier-notice tests into small functions (<50 lines). Refs #3867 Assisted-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
a08d5da8c9
commit
846a561a7b
4 files changed
+85
-72
No files matched your search
@@ -12,7 +12,7 @@
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const { getSessionsDir, getDateString, getTimeString, getSessionIdShort, sanitizeSessionId, getProjectName, getRepoIdentity, ensureDir, readFile, writeFile, runCommand, stripAnsi, log } = require('../lib/utils');
|
||||
const { generateSessionSummary, getContextRemainingPct, getContextThreshold } = require('../lib/llm-summary');
|
||||
const { generateSessionSummary, getContextRemainingPct, getContextThreshold, isClassifierNotice } = require('../lib/llm-summary');
|
||||
|
||||
const SUMMARY_START_MARKER = '<!-- ECC:SUMMARY:START -->';
|
||||
const SUMMARY_END_MARKER = '<!-- ECC:SUMMARY:END -->';
|
||||
@@ -20,8 +20,6 @@ const SESSION_SEPARATOR = '\n---\n';
|
||||
|
||||
// Harness noise: local command echoes, caveats, system reminders.
|
||||
const HARNESS_NOISE = /^<(local-command-caveat|local-command-stdout|command-name|command-message|command-args|system-reminder|task-notification)/i;
|
||||
// Harness notice shown after a safety-classifier stop; it is not a user ask (#3867).
|
||||
const CLASSIFIER_NOTICE = /^Your response above was stopped by a safety classifier/i;
|
||||
const TRANSCRIPT_UUID = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i;
|
||||
const NEXT_SESSION_FOOTER = '\n\n### Notes for Next Session\n-\n\n### Context to Load\n```\n[relevant files]\n```';
|
||||
const EMPTY_SESSION_BODY = '## Current State\n\n[Session context goes here]\n\n### Completed\n- [ ]\n\n### In Progress\n- [ ]\n\n### Notes for Next Session\n-\n\n### Context to Load\n```\n[relevant files]\n```';
|
||||
@@ -40,7 +38,7 @@ function userMessageText(entry) {
|
||||
// `isMeta` is also used for genuine channel- and plugin-originated
|
||||
// human prompts. Exclude known structured noise instead of
|
||||
// discarding every metadata-marked user turn.
|
||||
return HARNESS_NOISE.test(cleaned) || CLASSIFIER_NOTICE.test(cleaned) ? '' : cleaned;
|
||||
return HARNESS_NOISE.test(cleaned) || isClassifierNotice(cleaned) ? '' : cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -20,6 +20,8 @@ const MAX_TURNS = 25;
|
||||
const LLM_TIMEOUT_MS = 90000;
|
||||
// Leave the Stop hook time to persist its mechanical fallback after a timeout.
|
||||
const HOOK_FINISH_RESERVE_MS = 1000;
|
||||
// Harness notice shown after a safety-classifier stop; it is not a user ask (#3867).
|
||||
const CLASSIFIER_NOTICE = /^Your response above was stopped by a safety classifier/i;
|
||||
|
||||
function getLLMModel() {
|
||||
return process.env.ECC_LLM_SUMMARY_MODEL || 'haiku';
|
||||
@@ -30,6 +32,15 @@ function getContextThreshold() {
|
||||
return Number.isFinite(raw) && raw > 0 && raw <= 100 ? raw : 20;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a user-turn text is the harness notice left after a
|
||||
* safety-classifier stop. Anchored to the start so user messages that only
|
||||
* quote the notice are kept.
|
||||
*/
|
||||
function isClassifierNotice(text) {
|
||||
return CLASSIFIER_NOTICE.test(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the last MAX_TURNS user+assistant turns from a JSONL transcript.
|
||||
* Returns null when the transcript is missing or has no parseable turns.
|
||||
@@ -63,7 +74,7 @@ function extractConversationText(transcriptPath) {
|
||||
.join(' ')
|
||||
: '';
|
||||
const cleaned = text.replace(/\n+/g, ' ').trim();
|
||||
if (cleaned) {
|
||||
if (cleaned && !isClassifierNotice(cleaned)) {
|
||||
turns.push({ role: 'User', text: cleaned.slice(0, 400) });
|
||||
}
|
||||
}
|
||||
@@ -201,4 +212,4 @@ function generateSessionSummary(transcriptPath, { onSkip = () => {} } = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { generateSessionSummary, extractConversationText, getContextRemainingPct, getContextThreshold, getLLMModel };
|
||||
module.exports = { generateSessionSummary, extractConversationText, isClassifierNotice, getContextRemainingPct, getContextThreshold, getLLMModel };
|
||||
@@ -59,72 +59,55 @@ function sessionFileFor(home, uuid, date = getDateString()) {
|
||||
return path.join(home, '.claude', 'session-data', `${date}-${shortId}-session.tmp`);
|
||||
}
|
||||
|
||||
// Regression (#3867): the harness notice after a safety-classifier stop is
|
||||
// recorded as a user entry; it must not become a task in the summary.
|
||||
const CLASSIFIER_NOTICE_TEXT = 'Your response above was stopped by a safety classifier — this is not a tool or API error.';
|
||||
|
||||
function withTranscript(uuid, lines, fn) {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-'));
|
||||
try {
|
||||
const transcript = path.join(home, `${uuid}.jsonl`);
|
||||
fs.writeFileSync(transcript, lines.join('\n') + '\n');
|
||||
const res = runHook(home, transcript);
|
||||
assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`);
|
||||
fn(fs.readFileSync(sessionFileFor(home, uuid), 'utf8'));
|
||||
} finally {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function testDropsClassifierNotice() {
|
||||
withTranscript('3867aaaa-bbbb-4ccc-8ddd-eeeeeeee3867', [
|
||||
JSON.stringify({ type: 'user', message: { role: 'user', content: 'Refactor the config loader' } }),
|
||||
JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: CLASSIFIER_NOTICE_TEXT }] } }),
|
||||
], out => {
|
||||
assert.ok(out.includes('- Refactor the config loader'), `expected the real user task in:\n${out}`);
|
||||
assert.ok(!out.includes('safety classifier'), `classifier notice must not be summarized:\n${out}`);
|
||||
});
|
||||
}
|
||||
|
||||
function testNoticeOnlyTranscript() {
|
||||
withTranscript('3867bbbb-cccc-4ddd-8eee-ffffffff3867', [JSON.stringify({ type: 'user', content: CLASSIFIER_NOTICE_TEXT })], out => {
|
||||
assert.ok(!out.includes(START), `a notice-only transcript must not produce a summary block:\n${out}`);
|
||||
assert.ok(!out.includes('safety classifier'), `classifier notice must not be persisted:\n${out}`);
|
||||
});
|
||||
}
|
||||
|
||||
function testKeepsMessageQuotingNotice() {
|
||||
const userText = 'Why did I see "Your response above was stopped by a safety classifier" earlier?';
|
||||
withTranscript('3867cccc-dddd-4eee-8fff-aaaaaaaa3867', [JSON.stringify({ type: 'user', content: userText })], out => {
|
||||
assert.ok(out.includes(userText), `a real question about the notice must be kept:\n${out}`);
|
||||
});
|
||||
}
|
||||
|
||||
function runClassifierNoticeTests() {
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
// Regression (#3867): the harness notice after a safety-classifier stop is
|
||||
// recorded as a user entry; it must not become a task in the summary.
|
||||
const CLASSIFIER_NOTICE_TEXT = 'Your response above was stopped by a safety classifier — this is not a tool or API error.';
|
||||
|
||||
(test('drops safety-classifier notices from summary tasks', () => {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-'));
|
||||
try {
|
||||
const uuid = '3867aaaa-bbbb-4ccc-8ddd-eeeeeeee3867';
|
||||
const transcript = path.join(home, `${uuid}.jsonl`);
|
||||
fs.writeFileSync(
|
||||
transcript,
|
||||
[
|
||||
JSON.stringify({ type: 'user', message: { role: 'user', content: 'Refactor the config loader' } }),
|
||||
JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: CLASSIFIER_NOTICE_TEXT }] } }),
|
||||
].join('\n') + '\n'
|
||||
);
|
||||
|
||||
const res = runHook(home, transcript);
|
||||
assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`);
|
||||
const out = fs.readFileSync(sessionFileFor(home, uuid), 'utf8');
|
||||
assert.ok(out.includes('- Refactor the config loader'), `expected the real user task in:\n${out}`);
|
||||
assert.ok(!out.includes('safety classifier'), `classifier notice must not be summarized:\n${out}`);
|
||||
} finally {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
}) ? passed++ : failed++);
|
||||
|
||||
(test('writes no summary for a transcript holding only a classifier notice', () => {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-'));
|
||||
try {
|
||||
const uuid = '3867bbbb-cccc-4ddd-8eee-ffffffff3867';
|
||||
const transcript = path.join(home, `${uuid}.jsonl`);
|
||||
fs.writeFileSync(transcript, JSON.stringify({ type: 'user', content: CLASSIFIER_NOTICE_TEXT }) + '\n');
|
||||
|
||||
const res = runHook(home, transcript);
|
||||
assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`);
|
||||
const out = fs.readFileSync(sessionFileFor(home, uuid), 'utf8');
|
||||
assert.ok(!out.includes(START), `a notice-only transcript must not produce a summary block:\n${out}`);
|
||||
assert.ok(!out.includes('safety classifier'), `classifier notice must not be persisted:\n${out}`);
|
||||
} finally {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
}) ? passed++ : failed++);
|
||||
|
||||
(test('keeps a user message that only mentions the classifier notice', () => {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-'));
|
||||
try {
|
||||
const uuid = '3867cccc-dddd-4eee-8fff-aaaaaaaa3867';
|
||||
const transcript = path.join(home, `${uuid}.jsonl`);
|
||||
const userText = 'Why did I see "Your response above was stopped by a safety classifier" earlier?';
|
||||
fs.writeFileSync(transcript, JSON.stringify({ type: 'user', content: userText }) + '\n');
|
||||
|
||||
const res = runHook(home, transcript);
|
||||
assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`);
|
||||
const out = fs.readFileSync(sessionFileFor(home, uuid), 'utf8');
|
||||
assert.ok(out.includes(userText), `a real question about the notice must be kept:\n${out}`);
|
||||
} finally {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
}) ? passed++ : failed++);
|
||||
|
||||
return { passed, failed };
|
||||
const results = [
|
||||
test('drops safety-classifier notices from summary tasks', testDropsClassifierNotice),
|
||||
test('writes no summary for a transcript holding only a classifier notice', testNoticeOnlyTranscript),
|
||||
test('keeps a user message that only mentions the classifier notice', testKeepsMessageQuotingNotice),
|
||||
];
|
||||
const passed = results.filter(Boolean).length;
|
||||
return { passed, failed: results.length - passed };
|
||||
}
|
||||
|
||||
function runTests() {
|
||||
|
||||
@@ -10,7 +10,7 @@ const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
|
||||
const { extractConversationText, getContextRemainingPct, getContextThreshold, getLLMModel, generateSessionSummary } = require('../../scripts/lib/llm-summary');
|
||||
const { extractConversationText, isClassifierNotice, getContextRemainingPct, getContextThreshold, getLLMModel, generateSessionSummary } = require('../../scripts/lib/llm-summary');
|
||||
|
||||
console.log('=== Testing llm-summary.js ===\n');
|
||||
|
||||
@@ -154,6 +154,27 @@ test('collapses newlines to spaces', () => {
|
||||
assert.ok(result.includes('line one line two'));
|
||||
});
|
||||
|
||||
// Regression (#3867): the harness notice after a safety-classifier stop is
|
||||
// recorded as a user entry; it must not reach the LLM summary input.
|
||||
const CLASSIFIER_NOTICE_TEXT = 'Your response above was stopped by a safety classifier — this is not a tool or API error.';
|
||||
|
||||
test('drops safety-classifier notices from the LLM input', () => {
|
||||
const p = writeTranscript([userEntry('Refactor the config loader'), userEntry(CLASSIFIER_NOTICE_TEXT), assistantEntry('Done.')]);
|
||||
const result = extractConversationText(p);
|
||||
assert.ok(result.includes('Refactor the config loader'));
|
||||
assert.ok(!result.includes('safety classifier'), `classifier notice must not reach the LLM:\n${result}`);
|
||||
});
|
||||
|
||||
test('returns null when the only user turn is a classifier notice', () => {
|
||||
const p = writeTranscript([userEntry(CLASSIFIER_NOTICE_TEXT)]);
|
||||
assert.strictEqual(extractConversationText(p), null);
|
||||
});
|
||||
|
||||
test('isClassifierNotice matches only a leading notice', () => {
|
||||
assert.strictEqual(isClassifierNotice(CLASSIFIER_NOTICE_TEXT), true);
|
||||
assert.strictEqual(isClassifierNotice('Why did I see "Your response above was stopped by a safety classifier"?'), false);
|
||||
});
|
||||
|
||||
// --- getContextRemainingPct ---
|
||||
console.log('\ngetContextRemainingPct:');
|
||||
|
||||
|
||||
Reference in new issue
Block a user