maint: integrate reviewed contribution #3568

manifests/context-packs/skill-triggers@1.json

@@ -10,7 +10,7 @@
     "id": "hand-seeded",
     "source": "manual-curation-pending-regeneration"
   },
-  "registryDigest": "f0dfb40d0cf68d094aca58265074ac5e10a9138f865b9aabc8b302f668dfc2d8",
+  "registryDigest": "e801f7fb263a9489a5f499bf09153a067c7529fa32929d1a91a6c92bc4d1e0f9",
   "schemaVersion": 1,
   "triggers": {
     "skill:agent-runtime-gateway-smoke-test": [

skills/continuous-learning-v2/hooks/observe.sh

@@ -272,11 +272,24 @@ import json, sys, os, re
 # schemes (instead of a generic [A-Za-z]+\s+ that overlapped the value class)
 # prevent the catastrophic backtracking that pegged python at 100% CPU (#2278).
 _SECRET_RE = re.compile(
-    r"(?i)(api[_-]?key|token|secret|password|authorization|credentials?|auth)"
+    r"(?i)((?:api[_-]?key|token|secret|password|authorization|credentials?|auth)(?:_access_key|_key)?)"
     r"""(["'"'"'\s:=\\]{1,8})"""
     r"((?:bearer|basic|token|bot)\s+)?"
     r"([A-Za-z0-9_\-/.+=]{8,256})"
 )
+# Value-shaped secrets that carry no keyword: PEM private key blocks (to END or
+# end of the truncated text), GitHub tokens, AWS key IDs, and DSN passwords
+# (scheme://user:PASSWORD@host keeps scheme and user). Each branch is anchored
+# on a literal prefix and uses bounded or non-overlapping classes (#2278).
+# Tokens may follow a JSON escape (\n, \t, \uXXXX) since dict I/O is encoded
+# before scrubbing; a DSN cut by truncation before its "@" ends at \Z,
+# except a bare trailing host:port after http(s):// such as http://localhost:3000;
+# any other scheme (postgres://app:1234) is redacted even when the password is numeric (#3568).
+_SECRET_VALUE_RE = re.compile(
+    r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)"
+    r"|(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
+    r"""|(?<=://)(?:(?<=https://)|(?<=http://)|(?<!https://)(?<!http://)())([^\s:/@"\\]{1,256}:)(?(1)|(?!\d{1,5}\Z))[^\s@/"\\]{1,256}(?=@|\Z)"""
+)
 
 import signal
 def _clv2_bail(*_):
@@ -290,6 +303,7 @@ except Exception:
 
 raw = sys.stdin.read()[:2000]
 raw = _SECRET_RE.sub(lambda m: m.group(1) + m.group(2) + (m.group(3) or "") + "[REDACTED]", raw)
+raw = _SECRET_VALUE_RE.sub(lambda m: (m.group(2) or "") + "[REDACTED]", raw)
 print(json.dumps({"timestamp": os.environ["TIMESTAMP"], "event": "parse_error", "raw": raw}))
 ' >> "$OBSERVATIONS_FILE"
   exit 0
@@ -343,16 +357,30 @@ observation = {
 # schemes (instead of a generic [A-Za-z]+\s+ that overlapped the value class)
 # prevent the catastrophic backtracking that pegged python at 100% CPU (#2278).
 _SECRET_RE = re.compile(
-    r"(?i)(api[_-]?key|token|secret|password|authorization|credentials?|auth)"
+    r"(?i)((?:api[_-]?key|token|secret|password|authorization|credentials?|auth)(?:_access_key|_key)?)"
     r"""(["'"'"'\s:=\\]{1,8})"""
     r"((?:bearer|basic|token|bot)\s+)?"
     r"([A-Za-z0-9_\-/.+=]{8,256})"
 )
+# Value-shaped secrets that carry no keyword: PEM private key blocks (to END or
+# end of the truncated text), GitHub tokens, AWS key IDs, and DSN passwords
+# (scheme://user:PASSWORD@host keeps scheme and user). Each branch is anchored
+# on a literal prefix and uses bounded or non-overlapping classes (#2278).
+# Tokens may follow a JSON escape (\n, \t, \uXXXX) since dict I/O is encoded
+# before scrubbing; a DSN cut by truncation before its "@" ends at \Z,
+# except a bare trailing host:port after http(s):// such as http://localhost:3000;
+# any other scheme (postgres://app:1234) is redacted even when the password is numeric (#3568).
+_SECRET_VALUE_RE = re.compile(
+    r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)"
+    r"|(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
+    r"""|(?<=://)(?:(?<=https://)|(?<=http://)|(?<!https://)(?<!http://)())([^\s:/@"\\]{1,256}:)(?(1)|(?!\d{1,5}\Z))[^\s@/"\\]{1,256}(?=@|\Z)"""
+)
 
 def scrub(val):
     if val is None:
         return None
-    return _SECRET_RE.sub(lambda m: m.group(1) + m.group(2) + (m.group(3) or "") + "[REDACTED]", str(val))
+    val = _SECRET_RE.sub(lambda m: m.group(1) + m.group(2) + (m.group(3) or "") + "[REDACTED]", str(val))
+    return _SECRET_VALUE_RE.sub(lambda m: (m.group(2) or "") + "[REDACTED]", val)
 
 if parsed["input"]:
     observation["input"] = scrub(parsed["input"])

tests/hooks/observe-secret-redaction.test.js

@@ -16,13 +16,14 @@ function test(name, fn) {
   catch (error) { console.log(`  FAIL ${name}: ${error.message}`); failed += 1; }
 }
 
-function capture(phase, input) {
+function capture(phase, input, expectedEvent) {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-observe-redaction-'));
   try {
     const data = path.join(root, 'data');
     const result = spawnSync(process.execPath, [runner, `${phase}:observe`,
       'scripts/hooks/observe-runner.js', 'standard,strict'], {
-      input: JSON.stringify({ cwd: root, session_id: 'redaction-fixture', ...input }),
+      input: typeof input === 'string' ? input
+        : JSON.stringify({ cwd: root, session_id: 'redaction-fixture', ...input }),
       cwd: root, encoding: 'utf8', timeout: 12000,
       env: { ...process.env, HOME: root, USERPROFILE: root,
         CLV2_HOMUNCULUS_DIR: data, CLAUDE_CODE_ENTRYPOINT: 'cli',
@@ -33,7 +34,8 @@ function capture(phase, input) {
     assert.doesNotMatch(result.stderr, /runner failed|timed out/i);
     const records = fs.readFileSync(path.join(data, 'observations.jsonl'), 'utf8');
     assert.equal(records.trim().split('\n').length, 1);
-    assert.equal(JSON.parse(records).event, phase === 'pre' ? 'tool_start' : 'tool_complete');
+    assert.equal(JSON.parse(records).event,
+      expectedEvent || (phase === 'pre' ? 'tool_start' : 'tool_complete'));
     return records;
   } finally {
     fs.rmSync(root, { recursive: true, force: true });
@@ -48,25 +50,118 @@ if (process.platform === 'win32') {
   failed += 1;
 } else {
   // Synthetic canary data proves that captured credential-shaped text is masked.
-  for (const [label, content] of [
+  const canary = 'CANARY_TOKEN_abcdefghijk';
+  const pemCanary = 'MIIEcanaryPEMbodyAAAA';
+  for (const [label, content, secret = canary] of [
     ['double-quoted source', 'const apiKey = "CANARY_TOKEN_abcdefghijk";'],
     ['single-quoted source', "const password = 'CANARY_TOKEN_abcdefghijk';"],
     ['JSON credential', '{"token":"CANARY_TOKEN_abcdefghijk"}'],
-    ['nested escaped source', 'const auth = \\"CANARY_TOKEN_abcdefghijk\\";']
+    ['nested escaped source', 'const auth = \\"CANARY_TOKEN_abcdefghijk\\";'],
+    ['compound env var name', 'AWS_SECRET_ACCESS_KEY=CANARY/K7MDENG/bPxRfiCYEXAMPLEKEY',
+      'CANARY/K7MDENG/bPxRfiCYEXAMPLEKEY'],
+    ['lowercase compound name', 'aws_secret_access_key = CANARY/K7MDENG/bPxRfiCYEXAMPLEKEY',
+      'CANARY/K7MDENG/bPxRfiCYEXAMPLEKEY'],
+    ['DSN password', 'DATABASE_URL=postgres://app:CANARYdsnPassw0rd@db.internal:5432/app',
+      'CANARYdsnPassw0rd'],
+    ['PEM private key block', `-----BEGIN RSA ${'PRIVATE'} KEY-----\n${pemCanary}\n-----END RSA PRIVATE KEY-----`,
+      pemCanary],
+    ['truncated PEM private key block', `-----BEGIN OPENSSH ${'PRIVATE'} KEY-----\n${pemCanary}`, pemCanary],
+    ['GitHub classic token', `GH_PAT=${'ghp_'}CANARY${'0'.repeat(30)}`, `CANARY${'0'.repeat(30)}`],
+    ['GitHub fine-grained token', `use ${'github_pat_'}CANARY_${'1'.repeat(40)}`, `CANARY_${'1'.repeat(40)}`],
+    ['AWS access key id', `key id ${'AKIA'}CANARY0EXAMPLE00`, 'CANARY0EXAMPLE00']
   ]) {
     test(`redacts ${label} before persisting tool input`, () => {
       const records = capture('pre', { tool_name: 'Write',
         tool_input: { file_path: '/tmp/example.js', content } });
-      assert.ok(!records.includes('CANARY_TOKEN_abcdefghijk'), 'seeded credential persisted');
+      assert.ok(!records.includes(secret), 'seeded credential persisted');
       assert.ok(records.includes('[REDACTED]'));
     });
     test(`redacts ${label} before persisting structured tool output`, () => {
       const records = capture('post', { tool_name: 'Read',
         tool_input: { file_path: '/tmp/example.js' }, tool_response: { content } });
-      assert.ok(!records.includes('CANARY_TOKEN_abcdefghijk'), 'seeded credential persisted');
+      assert.ok(!records.includes(secret), 'seeded credential persisted');
       assert.ok(records.includes('[REDACTED]'));
     });
   }
+
+  // Structured output is JSON-encoded before scrubbing, so a newline before a
+  // bare token reaches the matcher as a literal backslash-n (greptile, #3568).
+  for (const [label, content, secret] of [
+    ['GitHub classic token after newline', `log line\n${'ghp_'}CANARY${'0'.repeat(30)}`,
+      `CANARY${'0'.repeat(30)}`],
+    ['GitHub fine-grained token after tab', `col\t${'github_pat_'}CANARY_${'1'.repeat(40)}`,
+      `CANARY_${'1'.repeat(40)}`],
+    ['AWS access key id after CRLF', `id:\r\n${'AKIA'}CANARY0EXAMPLE00`, 'CANARY0EXAMPLE00'],
+    ['AWS session key id after newline', `id:\n${'ASIA'}CANARY0EXAMPLE00`, 'CANARY0EXAMPLE00']
+  ]) {
+    test(`redacts ${label} in structured tool output`, () => {
+      const records = capture('post', { tool_name: 'Read',
+        tool_input: { file_path: '/tmp/example.js' }, tool_response: { content } });
+      assert.ok(!records.includes(secret), 'seeded credential persisted');
+      assert.ok(records.includes('[REDACTED]'));
+    });
+  }
+
+  // Capture truncation (5000 chars, 2000 in the parse-error fallback) can cut a
+  // DSN right before its '@'; the password must still be redacted (#3568).
+  const cutSecret = 'CANARYcutPassw0rd';
+  const dsnCutAt = (offset) => `${'a'.repeat(offset)} postgres://app:${cutSecret}@db.internal/app`;
+  const dsnPad = (atIndex) => atIndex - ' postgres://app:'.length - cutSecret.length;
+  for (const [label, phase, input, expectedEvent] of [
+    ['string tool output', 'post', { tool_name: 'Bash', tool_input: { command: 'env' },
+      tool_response: dsnCutAt(dsnPad(5000)) }],
+    // json.dumps prefixes '{"content": "' (13 chars) before the value.
+    ['structured tool output', 'post', { tool_name: 'Read', tool_input: { file_path: '/tmp/x' },
+      tool_response: { content: dsnCutAt(dsnPad(5000 - 13)) } }],
+    ['structured tool input', 'pre', { tool_name: 'Write',
+      tool_input: { content: dsnCutAt(dsnPad(5000 - 13)) } }],
+    // A top-level array is valid JSON but not a hook payload: parse-error
+    // fallback, which persists the first 2000 raw chars ('["' prefix).
+    ['parse-error fallback', 'post', JSON.stringify([dsnCutAt(dsnPad(2000 - 2))]), 'parse_error']
+  ]) {
+    test(`redacts DSN password truncated before '@' in ${label}`, () => {
+      const records = capture(phase, input, expectedEvent);
+      assert.ok(!records.includes(cutSecret), 'truncated DSN password persisted');
+      assert.ok(records.includes('postgres://app:[REDACTED]'));
+    });
+  }
+
+  // Numeric password cut right before '@' (or at end of text) must not be mistaken for host:port.
+  const numPad = (end) => 'a'.repeat(end - ' postgres://app:1234'.length) + ' postgres://app:1234';
+  for (const [label, phase, input, expectedEvent] of [
+    ['string tool output', 'post', { tool_name: 'Bash', tool_input: { command: 'env' },
+      tool_response: numPad(5000) }],
+    ['parse-error fallback', 'post', JSON.stringify([numPad(2000 - 2)]), 'parse_error']
+  ]) {
+    test(`redacts numeric DSN password cut at the truncation boundary in ${label}`, () => {
+      const records = capture(phase, input, expectedEvent);
+      assert.ok(!records.includes('app:1234'), 'numeric DSN password persisted');
+      assert.ok(records.includes('postgres://app:[REDACTED]'));
+    });
+  }
+
+  // Ordinary code, URLs without credentials and SSH remotes stay readable.
+  const benign = [
+    'const tokenizer = new Tokenizer(config);',
+    'max_tokens_per_request = 4096',
+    'token_count = countTokens(text)',
+    'password_hash = hashlib.sha256(pw)',
+    'fetch(http://localhost:8080/api/v1)',
+    'git clone git@github.com:org/repo.git',
+    'https://user@example.com/path'
+  ];
+  test('keeps ordinary code and credential-free URLs unredacted', () => {
+    const records = capture('post', { tool_name: 'Read',
+      tool_input: { file_path: '/tmp/example.js' }, tool_response: { content: benign.join('\n') } });
+    for (const line of benign) assert.ok(records.includes(line), `benign text altered: ${line}`);
+    assert.ok(!records.includes('[REDACTED]'));
+  });
+  test('keeps a trailing host:port in string output unredacted', () => {
+    const records = capture('post', { tool_name: 'Bash', tool_input: { command: 'npm run dev' },
+      tool_response: 'listening on http://localhost:3000' });
+    assert.ok(records.includes('http://localhost:3000'));
+    assert.ok(!records.includes('[REDACTED]'));
+  });
 }
 console.log(`Passed: ${passed}`);
 console.log(`Failed: ${failed}`);