diff --git a/manifests/context-packs/skill-triggers@1.json b/manifests/context-packs/skill-triggers@1.json index 39fcef132..f721e3486 100644 --- a/manifests/context-packs/skill-triggers@1.json +++ b/manifests/context-packs/skill-triggers@1.json @@ -10,7 +10,7 @@ "id": "hand-seeded", "source": "manual-curation-pending-regeneration" }, - "registryDigest": "f0dfb40d0cf68d094aca58265074ac5e10a9138f865b9aabc8b302f668dfc2d8", + "registryDigest": "e801f7fb263a9489a5f499bf09153a067c7529fa32929d1a91a6c92bc4d1e0f9", "schemaVersion": 1, "triggers": { "skill:agent-runtime-gateway-smoke-test": [ diff --git a/skills/continuous-learning-v2/hooks/observe.sh b/skills/continuous-learning-v2/hooks/observe.sh index ff0d3235f..aecb29a07 100755 --- a/skills/continuous-learning-v2/hooks/observe.sh +++ b/skills/continuous-learning-v2/hooks/observe.sh @@ -272,11 +272,24 @@ import json, sys, os, re # schemes (instead of a generic [A-Za-z]+\s+ that overlapped the value class) # prevent the catastrophic backtracking that pegged python at 100% CPU (#2278). _SECRET_RE = re.compile( - r"(?i)(api[_-]?key|token|secret|password|authorization|credentials?|auth)" + r"(?i)((?:api[_-]?key|token|secret|password|authorization|credentials?|auth)(?:_access_key|_key)?)" r"""(["'"'"'\s:=\\]{1,8})""" r"((?:bearer|basic|token|bot)\s+)?" r"([A-Za-z0-9_\-/.+=]{8,256})" ) +# Value-shaped secrets that carry no keyword: PEM private key blocks (to END or +# end of the truncated text), GitHub tokens, AWS key IDs, and DSN passwords +# (scheme://user:PASSWORD@host keeps scheme and user). Each branch is anchored +# on a literal prefix and uses bounded or non-overlapping classes (#2278). +# Tokens may follow a JSON escape (\n, \t, \uXXXX) since dict I/O is encoded +# before scrubbing; a DSN cut by truncation before its "@" ends at \Z, +# except a bare trailing host:port after http(s):// such as http://localhost:3000; +# any other scheme (postgres://app:1234) is redacted even when the password is numeric (#3568). +_SECRET_VALUE_RE = re.compile( + r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)" + r"|(?:(?> "$OBSERVATIONS_FILE" exit 0 @@ -343,16 +357,30 @@ observation = { # schemes (instead of a generic [A-Za-z]+\s+ that overlapped the value class) # prevent the catastrophic backtracking that pegged python at 100% CPU (#2278). _SECRET_RE = re.compile( - r"(?i)(api[_-]?key|token|secret|password|authorization|credentials?|auth)" + r"(?i)((?:api[_-]?key|token|secret|password|authorization|credentials?|auth)(?:_access_key|_key)?)" r"""(["'"'"'\s:=\\]{1,8})""" r"((?:bearer|basic|token|bot)\s+)?" r"([A-Za-z0-9_\-/.+=]{8,256})" ) +# Value-shaped secrets that carry no keyword: PEM private key blocks (to END or +# end of the truncated text), GitHub tokens, AWS key IDs, and DSN passwords +# (scheme://user:PASSWORD@host keeps scheme and user). Each branch is anchored +# on a literal prefix and uses bounded or non-overlapping classes (#2278). +# Tokens may follow a JSON escape (\n, \t, \uXXXX) since dict I/O is encoded +# before scrubbing; a DSN cut by truncation before its "@" ends at \Z, +# except a bare trailing host:port after http(s):// such as http://localhost:3000; +# any other scheme (postgres://app:1234) is redacted even when the password is numeric (#3568). +_SECRET_VALUE_RE = re.compile( + r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)" + r"|(?:(? { const records = capture('pre', { tool_name: 'Write', tool_input: { file_path: '/tmp/example.js', content } }); - assert.ok(!records.includes('CANARY_TOKEN_abcdefghijk'), 'seeded credential persisted'); + assert.ok(!records.includes(secret), 'seeded credential persisted'); assert.ok(records.includes('[REDACTED]')); }); test(`redacts ${label} before persisting structured tool output`, () => { const records = capture('post', { tool_name: 'Read', tool_input: { file_path: '/tmp/example.js' }, tool_response: { content } }); - assert.ok(!records.includes('CANARY_TOKEN_abcdefghijk'), 'seeded credential persisted'); + assert.ok(!records.includes(secret), 'seeded credential persisted'); assert.ok(records.includes('[REDACTED]')); }); } + + // Structured output is JSON-encoded before scrubbing, so a newline before a + // bare token reaches the matcher as a literal backslash-n (greptile, #3568). + for (const [label, content, secret] of [ + ['GitHub classic token after newline', `log line\n${'ghp_'}CANARY${'0'.repeat(30)}`, + `CANARY${'0'.repeat(30)}`], + ['GitHub fine-grained token after tab', `col\t${'github_pat_'}CANARY_${'1'.repeat(40)}`, + `CANARY_${'1'.repeat(40)}`], + ['AWS access key id after CRLF', `id:\r\n${'AKIA'}CANARY0EXAMPLE00`, 'CANARY0EXAMPLE00'], + ['AWS session key id after newline', `id:\n${'ASIA'}CANARY0EXAMPLE00`, 'CANARY0EXAMPLE00'] + ]) { + test(`redacts ${label} in structured tool output`, () => { + const records = capture('post', { tool_name: 'Read', + tool_input: { file_path: '/tmp/example.js' }, tool_response: { content } }); + assert.ok(!records.includes(secret), 'seeded credential persisted'); + assert.ok(records.includes('[REDACTED]')); + }); + } + + // Capture truncation (5000 chars, 2000 in the parse-error fallback) can cut a + // DSN right before its '@'; the password must still be redacted (#3568). + const cutSecret = 'CANARYcutPassw0rd'; + const dsnCutAt = (offset) => `${'a'.repeat(offset)} postgres://app:${cutSecret}@db.internal/app`; + const dsnPad = (atIndex) => atIndex - ' postgres://app:'.length - cutSecret.length; + for (const [label, phase, input, expectedEvent] of [ + ['string tool output', 'post', { tool_name: 'Bash', tool_input: { command: 'env' }, + tool_response: dsnCutAt(dsnPad(5000)) }], + // json.dumps prefixes '{"content": "' (13 chars) before the value. + ['structured tool output', 'post', { tool_name: 'Read', tool_input: { file_path: '/tmp/x' }, + tool_response: { content: dsnCutAt(dsnPad(5000 - 13)) } }], + ['structured tool input', 'pre', { tool_name: 'Write', + tool_input: { content: dsnCutAt(dsnPad(5000 - 13)) } }], + // A top-level array is valid JSON but not a hook payload: parse-error + // fallback, which persists the first 2000 raw chars ('["' prefix). + ['parse-error fallback', 'post', JSON.stringify([dsnCutAt(dsnPad(2000 - 2))]), 'parse_error'] + ]) { + test(`redacts DSN password truncated before '@' in ${label}`, () => { + const records = capture(phase, input, expectedEvent); + assert.ok(!records.includes(cutSecret), 'truncated DSN password persisted'); + assert.ok(records.includes('postgres://app:[REDACTED]')); + }); + } + + // Numeric password cut right before '@' (or at end of text) must not be mistaken for host:port. + const numPad = (end) => 'a'.repeat(end - ' postgres://app:1234'.length) + ' postgres://app:1234'; + for (const [label, phase, input, expectedEvent] of [ + ['string tool output', 'post', { tool_name: 'Bash', tool_input: { command: 'env' }, + tool_response: numPad(5000) }], + ['parse-error fallback', 'post', JSON.stringify([numPad(2000 - 2)]), 'parse_error'] + ]) { + test(`redacts numeric DSN password cut at the truncation boundary in ${label}`, () => { + const records = capture(phase, input, expectedEvent); + assert.ok(!records.includes('app:1234'), 'numeric DSN password persisted'); + assert.ok(records.includes('postgres://app:[REDACTED]')); + }); + } + + // Ordinary code, URLs without credentials and SSH remotes stay readable. + const benign = [ + 'const tokenizer = new Tokenizer(config);', + 'max_tokens_per_request = 4096', + 'token_count = countTokens(text)', + 'password_hash = hashlib.sha256(pw)', + 'fetch(http://localhost:8080/api/v1)', + 'git clone git@github.com:org/repo.git', + 'https://user@example.com/path' + ]; + test('keeps ordinary code and credential-free URLs unredacted', () => { + const records = capture('post', { tool_name: 'Read', + tool_input: { file_path: '/tmp/example.js' }, tool_response: { content: benign.join('\n') } }); + for (const line of benign) assert.ok(records.includes(line), `benign text altered: ${line}`); + assert.ok(!records.includes('[REDACTED]')); + }); + test('keeps a trailing host:port in string output unredacted', () => { + const records = capture('post', { tool_name: 'Bash', tool_input: { command: 'npm run dev' }, + tool_response: 'listening on http://localhost:3000' }); + assert.ok(records.includes('http://localhost:3000')); + assert.ok(!records.includes('[REDACTED]')); + }); } console.log(`Passed: ${passed}`); console.log(`Failed: ${failed}`);