fix(continuous-learning-v2): redact PEM blocks before the keyword matcher

With a label in front, as in secret=-----BEGIN RSA PRIVATE KEY-----, the
keyword matcher replaced only "-----BEGIN" and the PEM pattern could no
longer find its opening marker, so the key body was persisted. This was
already the case on main. Move the PEM pattern into _PEM_RE and run it
first in both the observation path and the parse-error fallback.
This commit is contained in:
Karim Masmoudi committed 2026-11-04 12:08:44 +01:00
1 parent caeb65b1ee
commit 0cc2864818
3 files changed
+14 -6

No files matched your search

@@ -10,7 +10,7 @@
"id": "hand-seeded",
"source": "manual-curation-pending-regeneration"
},
"registryDigest": "23d2319fbb6e5a75b914fd49baab8f8c1f7aab94513911918f3e71a6f96d0d92",
"registryDigest": "e97d914bd5d871e528da92d905fd768be77b89220a108817f59bac2cbc2f0969",
"schemaVersion": 1,
"triggers": {
"skill:agent-runtime-gateway-smoke-test": [
+10 -4
View File
@@ -290,9 +290,11 @@ _SECRET_RE = re.compile(
# before scrubbing; a DSN cut by truncation before its "@" ends at \Z,
# except a bare trailing host:port after http(s):// such as http://localhost:3000;
# any other scheme (postgres://app:1234) is redacted even when the password is numeric (#3568).
# PEM blocks run first: a label such as secret= in front of the BEGIN marker
# would otherwise let _SECRET_RE eat "-----BEGIN" and hide the block from this match.
_PEM_RE = re.compile(r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)")
_SECRET_VALUE_RE = re.compile(
r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)"
r"|(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
r"(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
r"|(?:(?<![A-Za-z0-9_-])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:sk-(?:ant|proj|svcacct|admin|or)-[A-Za-z0-9_-]{20,}|sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*)"
r"""|(?<=://)(?:(?<=https://)|(?<=http://)|(?<!https://)(?<!http://)())([^\s:/@"\\]{1,256}:)(?(1)|(?!\d{1,5}\Z))[^\s@/"\\]{1,256}(?=@|\Z)"""
)
@@ -315,6 +317,7 @@ except Exception:
raw_full = sys.stdin.read()
raw = raw_full[:2000]
raw = _PEM_RE.sub("[REDACTED]", raw)
raw = _SECRET_RE.sub(lambda m: m.group(1) + m.group(2) + (m.group(3) or "") + "[REDACTED]", raw)
raw = _SECRET_VALUE_RE.sub(lambda m: (m.group(2) or "") + "[REDACTED]", raw)
if len(raw_full) > 2000:
@@ -390,9 +393,11 @@ _SECRET_RE = re.compile(
# before scrubbing; a DSN cut by truncation before its "@" ends at \Z,
# except a bare trailing host:port after http(s):// such as http://localhost:3000;
# any other scheme (postgres://app:1234) is redacted even when the password is numeric (#3568).
# PEM blocks run first: a label such as secret= in front of the BEGIN marker
# would otherwise let _SECRET_RE eat "-----BEGIN" and hide the block from this match.
_PEM_RE = re.compile(r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)")
_SECRET_VALUE_RE = re.compile(
r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)"
r"|(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
r"(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
r"|(?:(?<![A-Za-z0-9_-])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:sk-(?:ant|proj|svcacct|admin|or)-[A-Za-z0-9_-]{20,}|sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])|eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]*)"
r"""|(?<=://)(?:(?<=https://)|(?<=http://)|(?<!https://)(?<!http://)())([^\s:/@"\\]{1,256}:)(?(1)|(?!\d{1,5}\Z))[^\s@/"\\]{1,256}(?=@|\Z)"""
)
@@ -409,6 +414,7 @@ def scrub(val):
val = str(val)
# Tool I/O is cut to 5000 chars before it reaches this process.
cut = len(val) >= 5000
val = _PEM_RE.sub("[REDACTED]", val)
val = _SECRET_RE.sub(lambda m: m.group(1) + m.group(2) + (m.group(3) or "") + "[REDACTED]", val)
val = _SECRET_VALUE_RE.sub(lambda m: (m.group(2) or "") + "[REDACTED]", val)
return _CUT_TAIL_RE.sub("[REDACTED]", val) if cut else val
+3 -1
View File
@@ -195,7 +195,9 @@ if (process.platform === 'win32') {
for (const [label, text, expected] of [
['long sk-proj key', `OPENAI_API_KEY=${openaiProjectKey}${'B'.repeat(300)}${longTail}`, 'OPENAI_API_KEY=[REDACTED]'],
['sk- password with a suffix', `password=${'sk-'}${'A'.repeat(30)}.${longTail}`, 'password=[REDACTED]'],
['GitHub token with a suffix', `token=${'ghp_'}${'0'.repeat(30)}/${longTail}`, 'token=[REDACTED]']
['GitHub token with a suffix', `token=${'ghp_'}${'0'.repeat(30)}/${longTail}`, 'token=[REDACTED]'],
['PEM private key block',
`secret=-----BEGIN RSA ${'PRIVATE'} KEY-----\n${longTail}\n-----END RSA PRIVATE KEY-----`, 'secret=[REDACTED]']
]) {
for (const [phase, input, expectedEvent] of [
['post', { tool_name: 'Read', tool_input: { file_path: '/tmp/x' }, tool_response: { content: text } }],