From f82eb1414ba10623decd0c2ea808acd28fb22c4b Mon Sep 17 00:00:00 2001 From: Muhammad Mirza Kurniawan Date: Sat, 31 Oct 2026 22:40:13 +0700 Subject: [PATCH] fix(continuous-learning): deliver v1 Stop nudge via native Stop contract behind explicit opt-in The v1 evaluator computed the learning signal but wrote it to stderr, which Claude Code never forwards to the model, so the learned-skills folder was never populated through this path. Delivery now follows the maintainer-endorsed shape: an explicit stop_nudge_opt_in flag (default off, so existing installs are unchanged), the native Stop block contract (decision block plus reason) instead of plain stdout text, at most one delivery per session via a temp-dir marker, Stop-loop protection via stop_hook_active, and delivery only for sessions with a registered lease from SessionStart. Tests: 7 new evaluator tests (opt-in off, no lease, one valid block then silence on repeat, stop_hook_active, short session, non-Stop event, missing session id). Evaluator suite 23/23 with the fix, failing on pristine; hooks suite 260/260; Stop stdout contract suite 48/48. Limits stated plainly: end-to-end delivery in a live Claude Code session is not certified from this host (unit-level spawn plus parse only), and the fix ships inert by default until a user opts in. --- scripts/hooks/evaluate-session.js | 111 +++++++++++- skills/continuous-learning/SKILL.md | 18 ++ skills/continuous-learning/config.json | 1 + tests/hooks/evaluate-session.test.js | 230 +++++++++++++++++++++++++ tests/hooks/hooks.test.js | 8 +- 5 files changed, 362 insertions(+), 6 deletions(-) diff --git a/scripts/hooks/evaluate-session.js b/scripts/hooks/evaluate-session.js index 3faa3891e..8c17c0d62 100644 --- a/scripts/hooks/evaluate-session.js +++ b/scripts/hooks/evaluate-session.js @@ -19,8 +19,23 @@ const { ensureDir, readFile, countInFile, - log + getTempDir, + log, + output } = require('../lib/utils'); +const { resolveHookSessionId } = require('../lib/hook-session'); +const { + resolveProjectContext, + getAliasContexts, + listSessionLeases, + resolveSessionId +} = require('../lib/observer-sessions'); + +// Marker that records a delivered Stop nudge, one per session, in the OS +// temp dir (same per-session temp-file pattern as suggest-compact's +// counters). A marker is what makes delivery once-per-session: a later +// user stop in the same session must not block again. +const DELIVERY_MARKER_PREFIX = 'claude-continuous-learning-delivered-'; // Read hook input from stdin (Claude Code provides transcript_path via stdin JSON) const MAX_STDIN = 1024 * 1024; @@ -42,10 +57,12 @@ process.stdin.on('end', () => { }); async function main() { - // Parse stdin JSON to get transcript_path + // Parse stdin JSON once; the parsed payload is reused by the delivery + // gates below (event name, stop_hook_active, cwd). + let input = null; let transcriptPath = null; try { - const input = JSON.parse(stdinData); + input = JSON.parse(stdinData); transcriptPath = input.transcript_path; } catch { // Fallback: try env var for backwards compatibility @@ -59,6 +76,10 @@ async function main() { // Default configuration let minSessionLength = 10; let learnedSkillsPath = getLearnedSkillsDir(); + // Delivery to the model is explicitly opt-in and off by default: a Stop + // hook that blocks the stop is user-visible behavior, so it must never + // turn on silently. + let stopNudgeOptIn = false; // Load config if exists const configContent = readFile(configFile); @@ -66,6 +87,7 @@ async function main() { try { const config = JSON.parse(configContent); minSessionLength = config.min_session_length ?? 10; + stopNudgeOptIn = config.stop_nudge_opt_in === true; if (config.learned_skills_path) { // Handle ~ in path @@ -92,9 +114,90 @@ async function main() { process.exit(0); } - // Signal to Claude that session should be evaluated for extractable patterns + // Signal to Claude that session should be evaluated for extractable patterns. + // The log lines below go to stderr (the debug log) only. Whether the + // model ever sees the signal is decided by deliverStopNudge, which fails + // closed: any gate miss means stderr only, exit 0, no stdout. log(`[ContinuousLearning] Session has ${messageCount} messages - evaluate for extractable patterns`); log(`[ContinuousLearning] Save learned skills to: ${learnedSkillsPath}`); + deliverStopNudge({ input, stdinData, messageCount, learnedSkillsPath, stopNudgeOptIn }); + process.exit(0); } + +/** + * Deliver the evaluation signal to the model, or deliver nothing. + * + * A Stop hook reaches the model only through the native Stop contract: + * decision "block" + reason, which continues the turn with the reason as + * context. Plain stdout text is not a delivery channel on Stop, so nothing + * else is ever written to stdout here. Every gate fails closed. + */ +function deliverStopNudge({ input, stdinData, messageCount, learnedSkillsPath, stopNudgeOptIn }) { + if (!stopNudgeOptIn) { + log('[ContinuousLearning] Stop nudge opt-in is off, skipping delivery'); + return; + } + // A payload that names a different event means this script is running + // under the wrong registration; an absent field is tolerated (older + // harnesses omit it), a mismatched one is not. + if (input && input.hook_event_name && input.hook_event_name !== 'Stop') { + log(`[ContinuousLearning] Skipping delivery: hook event is ${input.hook_event_name}, not Stop`); + return; + } + // Stop-loop protection: when a previous Stop already continued this stop, + // the hook is re-firing and must pass through, never block again. + if (input && input.stop_hook_active === true) { + log('[ContinuousLearning] Skipping delivery: Stop hook already fired for this stop'); + return; + } + // The payload session id identifies the harness conversation; env ids are + // only fallbacks, never preferred over it. + const sessionId = resolveSessionId(resolveHookSessionId(stdinData)); + if (!sessionId) { + log('[ContinuousLearning] Skipping delivery: no session id available'); + return; + } + if (!hasRegisteredSessionLease(input, sessionId)) { + log('[ContinuousLearning] Skipping delivery: no SessionStart lease for this session'); + return; + } + const markerFile = path.join(getTempDir(), `${DELIVERY_MARKER_PREFIX}${sessionId}`); + if (fs.existsSync(markerFile)) { + log('[ContinuousLearning] Stop nudge already delivered for this session, skipping'); + return; + } + try { + fs.writeFileSync(markerFile, `${new Date().toISOString()}\n`); + } catch (err) { + // The re-fired Stop (stop_hook_active) still bounds repeats to one extra + // turn, so deliver anyway rather than silently dropping the nudge. + log(`[ContinuousLearning] Warning: could not record delivery marker: ${err.message}`); + } + output({ + decision: 'block', + reason: `[ContinuousLearning] Session has ${messageCount} messages - evaluate for extractable patterns. Save learned skills to: ${learnedSkillsPath}` + }); +} + +/** + * The registered-native-lifecycle check: the session must hold a lease + * written by SessionStart under the ECC project context (including alias + * contexts for roots that moved mid-session). A Stop payload for a session + * that never went through SessionStart is not a registered ECC session, + * and its nudge must not reach the model. + */ +function hasRegisteredSessionLease(input, sessionId) { + try { + const cwd = (input && typeof input.cwd === 'string' && input.cwd) || process.cwd(); + const context = resolveProjectContext(cwd); + const contexts = [context, ...getAliasContexts(context)]; + return contexts.some(ctx => + listSessionLeases(ctx).some(file => path.basename(file) === `${sessionId}.json`) + ); + } catch (err) { + log(`[ContinuousLearning] Session lease check failed: ${err.message}`); + return false; + } +} diff --git a/skills/continuous-learning/SKILL.md b/skills/continuous-learning/SKILL.md index 9fe24a46e..ff26e31b6 100644 --- a/skills/continuous-learning/SKILL.md +++ b/skills/continuous-learning/SKILL.md @@ -90,6 +90,24 @@ Add to your `~/.claude/settings.json`: } ``` +## Stop Delivery (opt-in) + +By default the Stop hook only logs to stderr (the debug log); the model +never sees the signal. To actually deliver it, set +`"stop_nudge_opt_in": true` in `config.json`. Delivery then works through +the native Stop contract (`decision: "block"` + `reason`) and only when +every gate passes: + +- the session reached the message threshold; +- the session holds a lease written by SessionStart (the registered + native lifecycle check: unregistered sessions never deliver); +- this is a first Stop, not a re-fired one (`stop_hook_active` passes + through untouched, so the hook cannot loop); +- no nudge was already delivered for this session (once per session). + +Blocking the stop gives the model one extra turn to extract patterns and +save them; the next stop passes through and the session ends normally. + ## Why Stop Hook? - **Lightweight**: Runs once at session end diff --git a/skills/continuous-learning/config.json b/skills/continuous-learning/config.json index 1094b7e24..3c99c79e2 100644 --- a/skills/continuous-learning/config.json +++ b/skills/continuous-learning/config.json @@ -1,5 +1,6 @@ { "min_session_length": 10, + "stop_nudge_opt_in": false, "extraction_threshold": "medium", "auto_approve": false, "learned_skills_path": "~/.claude/skills/learned/", diff --git a/tests/hooks/evaluate-session.test.js b/tests/hooks/evaluate-session.test.js index 45a69c0b8..e9113590e 100644 --- a/tests/hooks/evaluate-session.test.js +++ b/tests/hooks/evaluate-session.test.js @@ -424,6 +424,236 @@ function runTests() { } })) passed++; else failed++; + // ── Stop delivery contract: opt-in + registered lease + loop protection ── + // The bug: the script logged its signal to stderr only, so the model never + // saw it. Delivery must emit exactly one valid Stop JSON payload, and only + // when every gate passes: explicit opt-in, a session lease registered + // through the native SessionStart lifecycle, a first (not re-fired) Stop, + // and no earlier delivery for this session. + console.log('\nStop delivery contract (opt-in + lease + loop protection):'); + + const observerSessions = require(path.join(__dirname, '..', '..', 'scripts', 'lib', 'observer-sessions')); + const deliveryConfigPath = path.join(__dirname, '..', '..', 'skills', 'continuous-learning', 'config.json'); + + // Test session ids below are already sanitized-safe (lowercase, digits, + // hyphens), so the delivery marker filename is a plain concatenation. + function deliveryMarkerPath(sessionId) { + return path.join(os.tmpdir(), `claude-continuous-learning-delivered-${sessionId}`); + } + + function runEvaluatePayload(payload, extraEnv = {}, dropKeys = []) { + const env = isolatedEnv(extraEnv); + for (const key of dropKeys) delete env[key]; + const result = spawnSync('node', [evaluateScript], { + encoding: 'utf8', + input: JSON.stringify(payload), + timeout: 10000, + env, + }); + return { + code: result.status || 0, + stdout: result.stdout || '', + stderr: result.stderr || '', + }; + } + + function withOptInConfig(value, fn) { + let originalContent = null; + try { + originalContent = fs.readFileSync(deliveryConfigPath, 'utf8'); + } catch { + // Config file may not exist + } + try { + const config = originalContent ? JSON.parse(originalContent) : {}; + config.stop_nudge_opt_in = value; + fs.writeFileSync(deliveryConfigPath, JSON.stringify(config, null, 2)); + return fn(); + } finally { + if (originalContent !== null) { + fs.writeFileSync(deliveryConfigPath, originalContent, 'utf8'); + } else { + try { fs.unlinkSync(deliveryConfigPath); } catch { /* best-effort */ } + } + } + } + + // Register a SessionStart lease for this session using the repo's own + // helpers, so the child process finds exactly what SessionStart writes. + function registerLeaseFor(workDir, homunculusDir, sessionId) { + const prevHomunculus = process.env.CLV2_HOMUNCULUS_DIR; + const prevProjectDir = process.env.CLAUDE_PROJECT_DIR; + process.env.CLV2_HOMUNCULUS_DIR = homunculusDir; + process.env.CLAUDE_PROJECT_DIR = workDir; + try { + const context = observerSessions.resolveProjectContext(workDir); + return observerSessions.writeSessionLease(context, sessionId, { hook: 'SessionStart' }); + } finally { + if (prevHomunculus === undefined) delete process.env.CLV2_HOMUNCULUS_DIR; + else process.env.CLV2_HOMUNCULUS_DIR = prevHomunculus; + if (prevProjectDir === undefined) delete process.env.CLAUDE_PROJECT_DIR; + else process.env.CLAUDE_PROJECT_DIR = prevProjectDir; + } + } + + function stopPayload(transcript, sessionId, workDir, overrides = {}) { + return { + session_id: sessionId, + transcript_path: transcript, + cwd: workDir, + hook_event_name: 'Stop', + stop_hook_active: false, + ...overrides, + }; + } + + if (test('emits no stdout when opt-in is off, even for a long session', () => { + const testDir = createTestDir(); + const transcript = createTranscript(testDir, 12); + try { + const result = runEvaluatePayload(stopPayload(transcript, 'delivery-optoff-1', testDir)); + assert.strictEqual(result.code, 0); + assert.strictEqual(result.stdout, '', 'Model must receive nothing when opt-in is off'); + assert.ok(result.stderr.includes('evaluate'), 'Stderr log line is unchanged'); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + + if (test('emits no stdout when opt-in is on but no session lease is registered', () => { + const testDir = createTestDir(); + const transcript = createTranscript(testDir, 12); + const homunculusDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-delivery-home-')); + try { + const result = withOptInConfig(true, () => runEvaluatePayload( + stopPayload(transcript, 'delivery-nolease-1', testDir), + { CLV2_HOMUNCULUS_DIR: homunculusDir, CLAUDE_PROJECT_DIR: testDir } + )); + assert.strictEqual(result.code, 0); + assert.strictEqual(result.stdout, '', 'Unregistered sessions must not reach the model'); + } finally { + cleanupTestDir(testDir); + fs.rmSync(homunculusDir, { recursive: true, force: true }); + } + })) passed++; else failed++; + + if (test('emits one valid Stop block JSON with opt-in + lease, then nothing on repeat', () => { + const testDir = createTestDir(); + const transcript = createTranscript(testDir, 12); + const homunculusDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-delivery-home-')); + const sessionId = 'delivery-ok-1'; + const marker = deliveryMarkerPath(sessionId); + try { + fs.rmSync(marker, { force: true }); + registerLeaseFor(testDir, homunculusDir, sessionId); + const env = { CLV2_HOMUNCULUS_DIR: homunculusDir, CLAUDE_PROJECT_DIR: testDir }; + const first = withOptInConfig(true, () => runEvaluatePayload( + stopPayload(transcript, sessionId, testDir), env + )); + assert.strictEqual(first.code, 0); + let parsed; + try { + parsed = JSON.parse(first.stdout); + } catch { + assert.fail(`Stdout must be a single valid JSON document. Got: ${first.stdout}`); + } + assert.strictEqual(parsed.decision, 'block', 'Native Stop delivery blocks the stop'); + assert.ok( + typeof parsed.reason === 'string' && parsed.reason.includes('12 messages'), + `Reason must carry the session signal. Got: ${parsed.reason}` + ); + assert.ok( + parsed.reason.includes('learned'), + `Reason must name where learned skills go. Got: ${parsed.reason}` + ); + const second = withOptInConfig(true, () => runEvaluatePayload( + stopPayload(transcript, sessionId, testDir), env + )); + assert.strictEqual(second.stdout, '', 'Second Stop for the same session must not re-deliver'); + } finally { + fs.rmSync(marker, { force: true }); + cleanupTestDir(testDir); + fs.rmSync(homunculusDir, { recursive: true, force: true }); + } + })) passed++; else failed++; + + if (test('emits no stdout when stop_hook_active is true, even with opt-in + lease', () => { + const testDir = createTestDir(); + const transcript = createTranscript(testDir, 12); + const homunculusDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-delivery-home-')); + const sessionId = 'delivery-loop-1'; + const marker = deliveryMarkerPath(sessionId); + try { + fs.rmSync(marker, { force: true }); + registerLeaseFor(testDir, homunculusDir, sessionId); + const result = withOptInConfig(true, () => runEvaluatePayload( + stopPayload(transcript, sessionId, testDir, { stop_hook_active: true }), + { CLV2_HOMUNCULUS_DIR: homunculusDir, CLAUDE_PROJECT_DIR: testDir } + )); + assert.strictEqual(result.code, 0); + assert.strictEqual(result.stdout, '', 'A re-fired Stop must pass through, never re-block'); + } finally { + fs.rmSync(marker, { force: true }); + cleanupTestDir(testDir); + fs.rmSync(homunculusDir, { recursive: true, force: true }); + } + })) passed++; else failed++; + + if (test('emits no stdout for a short session, even with opt-in + lease', () => { + const testDir = createTestDir(); + const transcript = createTranscript(testDir, 5); + const homunculusDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-delivery-home-')); + const sessionId = 'delivery-short-1'; + try { + registerLeaseFor(testDir, homunculusDir, sessionId); + const result = withOptInConfig(true, () => runEvaluatePayload( + stopPayload(transcript, sessionId, testDir), + { CLV2_HOMUNCULUS_DIR: homunculusDir, CLAUDE_PROJECT_DIR: testDir } + )); + assert.strictEqual(result.code, 0); + assert.strictEqual(result.stdout, '', 'Below-threshold sessions must not deliver'); + assert.ok(result.stderr.includes('too short'), 'Short session still logs as too short'); + } finally { + cleanupTestDir(testDir); + fs.rmSync(homunculusDir, { recursive: true, force: true }); + } + })) passed++; else failed++; + + if (test('emits no stdout when the payload event is not Stop', () => { + const testDir = createTestDir(); + const transcript = createTranscript(testDir, 12); + const homunculusDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-delivery-home-')); + const sessionId = 'delivery-wrongevent-1'; + try { + registerLeaseFor(testDir, homunculusDir, sessionId); + const result = withOptInConfig(true, () => runEvaluatePayload( + stopPayload(transcript, sessionId, testDir, { hook_event_name: 'PreToolUse' }), + { CLV2_HOMUNCULUS_DIR: homunculusDir, CLAUDE_PROJECT_DIR: testDir } + )); + assert.strictEqual(result.code, 0); + assert.strictEqual(result.stdout, '', 'Delivery is Stop-only; other events must not trigger it'); + } finally { + cleanupTestDir(testDir); + fs.rmSync(homunculusDir, { recursive: true, force: true }); + } + })) passed++; else failed++; + + if (test('emits no stdout when no session id is available, even with opt-in', () => { + const testDir = createTestDir(); + const transcript = createTranscript(testDir, 12); + try { + const payload = stopPayload(transcript, undefined, testDir); + delete payload.session_id; + const result = withOptInConfig(true, () => runEvaluatePayload( + payload, {}, ['CLAUDE_SESSION_ID', 'ECC_SESSION_ID'] + )); + assert.strictEqual(result.code, 0); + assert.strictEqual(result.stdout, '', 'Without a session id there is nothing to bind delivery to'); + } finally { + cleanupTestDir(testDir); + } + })) passed++; else failed++; + // Summary console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); process.exit(failed > 0 ? 1 : 0); diff --git a/tests/hooks/hooks.test.js b/tests/hooks/hooks.test.js index db469d34d..733ee2ac0 100644 --- a/tests/hooks/hooks.test.js +++ b/tests/hooks/hooks.test.js @@ -3950,13 +3950,17 @@ async function runTests() { // ─── Round 23: Bug fixes & high-priority gap coverage ─── // Helper: create a patched evaluate-session.js wrapper that resolves - // require('../lib/utils') to the real utils.js and uses a custom config path + // require('../lib/...') to the real lib files and uses a custom config path const realUtilsPath = path.resolve(__dirname, '..', '..', 'scripts', 'lib', 'utils.js'); + const realHookSessionPath = path.resolve(__dirname, '..', '..', 'scripts', 'lib', 'hook-session.js'); + const realObserverSessionsPath = path.resolve(__dirname, '..', '..', 'scripts', 'lib', 'observer-sessions.js'); function createEvalWrapper(testDir, configPath) { const wrapperScript = path.join(testDir, 'eval-wrapper.js'); let src = fs.readFileSync(path.join(scriptsDir, 'evaluate-session.js'), 'utf8'); - // Patch require to use absolute path (the temp dir doesn't have ../lib/utils) + // Patch requires to use absolute paths (the temp dir doesn't have ../lib/*) src = src.replace(/require\('\.\.\/lib\/utils'\)/, `require(${JSON.stringify(realUtilsPath)})`); + src = src.replace(/require\('\.\.\/lib\/hook-session'\)/, `require(${JSON.stringify(realHookSessionPath)})`); + src = src.replace(/require\('\.\.\/lib\/observer-sessions'\)/, `require(${JSON.stringify(realObserverSessionsPath)})`); // Patch config file path to point to our test config src = src.replace(/const configFile = path\.join\(scriptDir.*?config\.json'\);/, `const configFile = ${JSON.stringify(configPath)};`); fs.writeFileSync(wrapperScript, src);