From 846a561a7bc4c156409e296cb584ce8ea0d488f2 Mon Sep 17 00:00:00 2001 From: Rahul Selvaraj <55731288+Laihu08@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:06:51 +0800 Subject: [PATCH] fix(hooks): drop classifier notices from LLM summary input The safety-classifier notice filter only covered the mechanical summary. When generateSessionSummary runs (low context or message interval), extractConversationText read the raw transcript and passed the notice to the LLM as a user turn, so it could still reach saved context. Move the check into one shared isClassifierNotice() in scripts/lib/llm-summary.js and use it in both userMessageText() and extractConversationText(). Add LLM-path regression tests, and split the session-end classifier-notice tests into small functions (<50 lines). Refs #3867 Assisted-by: Claude --- scripts/hooks/session-end.js | 6 +- scripts/lib/llm-summary.js | 15 ++++- tests/hooks/session-end.test.js | 113 ++++++++++++++------------------ tests/lib/llm-summary.test.js | 23 ++++++- 4 files changed, 85 insertions(+), 72 deletions(-) diff --git a/scripts/hooks/session-end.js b/scripts/hooks/session-end.js index 63248ae36..ece98de23 100644 --- a/scripts/hooks/session-end.js +++ b/scripts/hooks/session-end.js @@ -12,7 +12,7 @@ const path = require('path'); const fs = require('fs'); const { getSessionsDir, getDateString, getTimeString, getSessionIdShort, sanitizeSessionId, getProjectName, getRepoIdentity, ensureDir, readFile, writeFile, runCommand, stripAnsi, log } = require('../lib/utils'); -const { generateSessionSummary, getContextRemainingPct, getContextThreshold } = require('../lib/llm-summary'); +const { generateSessionSummary, getContextRemainingPct, getContextThreshold, isClassifierNotice } = require('../lib/llm-summary'); const SUMMARY_START_MARKER = ''; const SUMMARY_END_MARKER = ''; @@ -20,8 +20,6 @@ const SESSION_SEPARATOR = '\n---\n'; // Harness noise: local command echoes, caveats, system reminders. const HARNESS_NOISE = /^<(local-command-caveat|local-command-stdout|command-name|command-message|command-args|system-reminder|task-notification)/i; -// Harness notice shown after a safety-classifier stop; it is not a user ask (#3867). -const CLASSIFIER_NOTICE = /^Your response above was stopped by a safety classifier/i; const TRANSCRIPT_UUID = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i; const NEXT_SESSION_FOOTER = '\n\n### Notes for Next Session\n-\n\n### Context to Load\n```\n[relevant files]\n```'; const EMPTY_SESSION_BODY = '## Current State\n\n[Session context goes here]\n\n### Completed\n- [ ]\n\n### In Progress\n- [ ]\n\n### Notes for Next Session\n-\n\n### Context to Load\n```\n[relevant files]\n```'; @@ -40,7 +38,7 @@ function userMessageText(entry) { // `isMeta` is also used for genuine channel- and plugin-originated // human prompts. Exclude known structured noise instead of // discarding every metadata-marked user turn. - return HARNESS_NOISE.test(cleaned) || CLASSIFIER_NOTICE.test(cleaned) ? '' : cleaned; + return HARNESS_NOISE.test(cleaned) || isClassifierNotice(cleaned) ? '' : cleaned; } /** diff --git a/scripts/lib/llm-summary.js b/scripts/lib/llm-summary.js index d9a1f006c..7a56211bc 100644 --- a/scripts/lib/llm-summary.js +++ b/scripts/lib/llm-summary.js @@ -20,6 +20,8 @@ const MAX_TURNS = 25; const LLM_TIMEOUT_MS = 90000; // Leave the Stop hook time to persist its mechanical fallback after a timeout. const HOOK_FINISH_RESERVE_MS = 1000; +// Harness notice shown after a safety-classifier stop; it is not a user ask (#3867). +const CLASSIFIER_NOTICE = /^Your response above was stopped by a safety classifier/i; function getLLMModel() { return process.env.ECC_LLM_SUMMARY_MODEL || 'haiku'; @@ -30,6 +32,15 @@ function getContextThreshold() { return Number.isFinite(raw) && raw > 0 && raw <= 100 ? raw : 20; } +/** + * True when a user-turn text is the harness notice left after a + * safety-classifier stop. Anchored to the start so user messages that only + * quote the notice are kept. + */ +function isClassifierNotice(text) { + return CLASSIFIER_NOTICE.test(text); +} + /** * Extract the last MAX_TURNS user+assistant turns from a JSONL transcript. * Returns null when the transcript is missing or has no parseable turns. @@ -63,7 +74,7 @@ function extractConversationText(transcriptPath) { .join(' ') : ''; const cleaned = text.replace(/\n+/g, ' ').trim(); - if (cleaned) { + if (cleaned && !isClassifierNotice(cleaned)) { turns.push({ role: 'User', text: cleaned.slice(0, 400) }); } } @@ -201,4 +212,4 @@ function generateSessionSummary(transcriptPath, { onSkip = () => {} } = {}) { } } -module.exports = { generateSessionSummary, extractConversationText, getContextRemainingPct, getContextThreshold, getLLMModel }; +module.exports = { generateSessionSummary, extractConversationText, isClassifierNotice, getContextRemainingPct, getContextThreshold, getLLMModel }; diff --git a/tests/hooks/session-end.test.js b/tests/hooks/session-end.test.js index 049e3c700..b09f84a84 100644 --- a/tests/hooks/session-end.test.js +++ b/tests/hooks/session-end.test.js @@ -59,72 +59,55 @@ function sessionFileFor(home, uuid, date = getDateString()) { return path.join(home, '.claude', 'session-data', `${date}-${shortId}-session.tmp`); } +// Regression (#3867): the harness notice after a safety-classifier stop is +// recorded as a user entry; it must not become a task in the summary. +const CLASSIFIER_NOTICE_TEXT = 'Your response above was stopped by a safety classifier — this is not a tool or API error.'; + +function withTranscript(uuid, lines, fn) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-')); + try { + const transcript = path.join(home, `${uuid}.jsonl`); + fs.writeFileSync(transcript, lines.join('\n') + '\n'); + const res = runHook(home, transcript); + assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`); + fn(fs.readFileSync(sessionFileFor(home, uuid), 'utf8')); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } +} + +function testDropsClassifierNotice() { + withTranscript('3867aaaa-bbbb-4ccc-8ddd-eeeeeeee3867', [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Refactor the config loader' } }), + JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: CLASSIFIER_NOTICE_TEXT }] } }), + ], out => { + assert.ok(out.includes('- Refactor the config loader'), `expected the real user task in:\n${out}`); + assert.ok(!out.includes('safety classifier'), `classifier notice must not be summarized:\n${out}`); + }); +} + +function testNoticeOnlyTranscript() { + withTranscript('3867bbbb-cccc-4ddd-8eee-ffffffff3867', [JSON.stringify({ type: 'user', content: CLASSIFIER_NOTICE_TEXT })], out => { + assert.ok(!out.includes(START), `a notice-only transcript must not produce a summary block:\n${out}`); + assert.ok(!out.includes('safety classifier'), `classifier notice must not be persisted:\n${out}`); + }); +} + +function testKeepsMessageQuotingNotice() { + const userText = 'Why did I see "Your response above was stopped by a safety classifier" earlier?'; + withTranscript('3867cccc-dddd-4eee-8fff-aaaaaaaa3867', [JSON.stringify({ type: 'user', content: userText })], out => { + assert.ok(out.includes(userText), `a real question about the notice must be kept:\n${out}`); + }); +} + function runClassifierNoticeTests() { - let passed = 0; - let failed = 0; - - // Regression (#3867): the harness notice after a safety-classifier stop is - // recorded as a user entry; it must not become a task in the summary. - const CLASSIFIER_NOTICE_TEXT = 'Your response above was stopped by a safety classifier — this is not a tool or API error.'; - - (test('drops safety-classifier notices from summary tasks', () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-')); - try { - const uuid = '3867aaaa-bbbb-4ccc-8ddd-eeeeeeee3867'; - const transcript = path.join(home, `${uuid}.jsonl`); - fs.writeFileSync( - transcript, - [ - JSON.stringify({ type: 'user', message: { role: 'user', content: 'Refactor the config loader' } }), - JSON.stringify({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: CLASSIFIER_NOTICE_TEXT }] } }), - ].join('\n') + '\n' - ); - - const res = runHook(home, transcript); - assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`); - const out = fs.readFileSync(sessionFileFor(home, uuid), 'utf8'); - assert.ok(out.includes('- Refactor the config loader'), `expected the real user task in:\n${out}`); - assert.ok(!out.includes('safety classifier'), `classifier notice must not be summarized:\n${out}`); - } finally { - fs.rmSync(home, { recursive: true, force: true }); - } - }) ? passed++ : failed++); - - (test('writes no summary for a transcript holding only a classifier notice', () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-')); - try { - const uuid = '3867bbbb-cccc-4ddd-8eee-ffffffff3867'; - const transcript = path.join(home, `${uuid}.jsonl`); - fs.writeFileSync(transcript, JSON.stringify({ type: 'user', content: CLASSIFIER_NOTICE_TEXT }) + '\n'); - - const res = runHook(home, transcript); - assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`); - const out = fs.readFileSync(sessionFileFor(home, uuid), 'utf8'); - assert.ok(!out.includes(START), `a notice-only transcript must not produce a summary block:\n${out}`); - assert.ok(!out.includes('safety classifier'), `classifier notice must not be persisted:\n${out}`); - } finally { - fs.rmSync(home, { recursive: true, force: true }); - } - }) ? passed++ : failed++); - - (test('keeps a user message that only mentions the classifier notice', () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-end-')); - try { - const uuid = '3867cccc-dddd-4eee-8fff-aaaaaaaa3867'; - const transcript = path.join(home, `${uuid}.jsonl`); - const userText = 'Why did I see "Your response above was stopped by a safety classifier" earlier?'; - fs.writeFileSync(transcript, JSON.stringify({ type: 'user', content: userText }) + '\n'); - - const res = runHook(home, transcript); - assert.strictEqual(res.status || 0, 0, `hook exited ${res.status}: ${res.stderr}`); - const out = fs.readFileSync(sessionFileFor(home, uuid), 'utf8'); - assert.ok(out.includes(userText), `a real question about the notice must be kept:\n${out}`); - } finally { - fs.rmSync(home, { recursive: true, force: true }); - } - }) ? passed++ : failed++); - - return { passed, failed }; + const results = [ + test('drops safety-classifier notices from summary tasks', testDropsClassifierNotice), + test('writes no summary for a transcript holding only a classifier notice', testNoticeOnlyTranscript), + test('keeps a user message that only mentions the classifier notice', testKeepsMessageQuotingNotice), + ]; + const passed = results.filter(Boolean).length; + return { passed, failed: results.length - passed }; } function runTests() { diff --git a/tests/lib/llm-summary.test.js b/tests/lib/llm-summary.test.js index 514ccedac..e8b3d67f8 100644 --- a/tests/lib/llm-summary.test.js +++ b/tests/lib/llm-summary.test.js @@ -10,7 +10,7 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); -const { extractConversationText, getContextRemainingPct, getContextThreshold, getLLMModel, generateSessionSummary } = require('../../scripts/lib/llm-summary'); +const { extractConversationText, isClassifierNotice, getContextRemainingPct, getContextThreshold, getLLMModel, generateSessionSummary } = require('../../scripts/lib/llm-summary'); console.log('=== Testing llm-summary.js ===\n'); @@ -154,6 +154,27 @@ test('collapses newlines to spaces', () => { assert.ok(result.includes('line one line two')); }); +// Regression (#3867): the harness notice after a safety-classifier stop is +// recorded as a user entry; it must not reach the LLM summary input. +const CLASSIFIER_NOTICE_TEXT = 'Your response above was stopped by a safety classifier — this is not a tool or API error.'; + +test('drops safety-classifier notices from the LLM input', () => { + const p = writeTranscript([userEntry('Refactor the config loader'), userEntry(CLASSIFIER_NOTICE_TEXT), assistantEntry('Done.')]); + const result = extractConversationText(p); + assert.ok(result.includes('Refactor the config loader')); + assert.ok(!result.includes('safety classifier'), `classifier notice must not reach the LLM:\n${result}`); +}); + +test('returns null when the only user turn is a classifier notice', () => { + const p = writeTranscript([userEntry(CLASSIFIER_NOTICE_TEXT)]); + assert.strictEqual(extractConversationText(p), null); +}); + +test('isClassifierNotice matches only a leading notice', () => { + assert.strictEqual(isClassifierNotice(CLASSIFIER_NOTICE_TEXT), true); + assert.strictEqual(isClassifierNotice('Why did I see "Your response above was stopped by a safety classifier"?'), false); +}); + // --- getContextRemainingPct --- console.log('\ngetContextRemainingPct:');