mirror of
https://github.com/affaan-m/ECC.git
synced 2026-11-17 13:32:18 +00:00
fix(continuous-learning-v2): redact sk- keys and JWTs cut by capture truncation
Observations are cut to 5000 chars (2000 in the parse-error fallback) before scrubbing. A JWT cut inside its payload lost its second dot and a key cut short of 20 body chars missed the length floor, so the remaining header, claims or key prefix were persisted. Accept a match that runs to the end of the capture, the same way the PEM and DSN branches do. Tests cover both cuts in string output, structured output and the parse-error fallback, and compare the JWT and Bash cases against their exact expected values.
This commit is contained in:
1 parent
61826bdc99
commit
b56f5e77b0
3 files changed
+47
-8
No files matched your search
@@ -10,7 +10,7 @@
|
||||
"id": "hand-seeded",
|
||||
"source": "manual-curation-pending-regeneration"
|
||||
},
|
||||
"registryDigest": "cf364a6536f8fb3b0f0cda52a49fd3751513eff988ce5fadc15abcb1e4577358",
|
||||
"registryDigest": "d065943b86358b5dc5f84c5910de117b21650c0be34071457d20820b28d68e7a",
|
||||
"schemaVersion": 1,
|
||||
"triggers": {
|
||||
"skill:agent-runtime-gateway-smoke-test": [
|
||||
|
||||
@@ -279,8 +279,9 @@ _SECRET_RE = re.compile(
|
||||
)
|
||||
# Value-shaped secrets that carry no keyword: PEM private key blocks (to END or
|
||||
# end of the truncated text), GitHub tokens, AWS key IDs, Anthropic and OpenAI
|
||||
# style sk- keys, JWTs (eyJ header, payload, optional signature) and DSN
|
||||
# passwords (scheme://user:PASSWORD@host keeps scheme and user). Each branch is anchored
|
||||
# style sk- keys and JWTs (eyJ header, payload, optional signature; either one
|
||||
# cut by truncation ends at \Z), and DSN passwords (scheme://user:PASSWORD@host
|
||||
# keeps scheme and user). Each branch is anchored
|
||||
# on a literal prefix and uses bounded or non-overlapping classes (#2278).
|
||||
# Tokens may follow a JSON escape (\n, \t, \uXXXX) since dict I/O is encoded
|
||||
# before scrubbing; a DSN cut by truncation before its "@" ends at \Z,
|
||||
@@ -289,7 +290,7 @@ _SECRET_RE = re.compile(
|
||||
_SECRET_VALUE_RE = re.compile(
|
||||
r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)"
|
||||
r"|(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
|
||||
r"|(?:(?<![A-Za-z0-9_-])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:sk-(?:ant|proj|svcacct|admin|or)-[A-Za-z0-9_-]{20,512}|sk-[A-Za-z0-9]{20,512}(?![A-Za-z0-9_-])|eyJ[A-Za-z0-9_-]{8,1024}\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{0,1024})"
|
||||
r"|(?:(?<![A-Za-z0-9_-])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:sk-(?:ant|proj|svcacct|admin|or)-(?:[A-Za-z0-9_-]{20,512}|[A-Za-z0-9_-]{0,19}\Z)|sk-(?:[A-Za-z0-9]{20,512}(?![A-Za-z0-9_-])|[A-Za-z0-9]{1,19}\Z)|eyJ[A-Za-z0-9_-]{8,1024}(?:\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{0,1024}|(?:\.[A-Za-z0-9_-]{0,8192})?\Z))"
|
||||
r"""|(?<=://)(?:(?<=https://)|(?<=http://)|(?<!https://)(?<!http://)())([^\s:/@"\\]{1,256}:)(?(1)|(?!\d{1,5}\Z))[^\s@/"\\]{1,256}(?=@|\Z)"""
|
||||
)
|
||||
|
||||
@@ -366,8 +367,9 @@ _SECRET_RE = re.compile(
|
||||
)
|
||||
# Value-shaped secrets that carry no keyword: PEM private key blocks (to END or
|
||||
# end of the truncated text), GitHub tokens, AWS key IDs, Anthropic and OpenAI
|
||||
# style sk- keys, JWTs (eyJ header, payload, optional signature) and DSN
|
||||
# passwords (scheme://user:PASSWORD@host keeps scheme and user). Each branch is anchored
|
||||
# style sk- keys and JWTs (eyJ header, payload, optional signature; either one
|
||||
# cut by truncation ends at \Z), and DSN passwords (scheme://user:PASSWORD@host
|
||||
# keeps scheme and user). Each branch is anchored
|
||||
# on a literal prefix and uses bounded or non-overlapping classes (#2278).
|
||||
# Tokens may follow a JSON escape (\n, \t, \uXXXX) since dict I/O is encoded
|
||||
# before scrubbing; a DSN cut by truncation before its "@" ends at \Z,
|
||||
@@ -376,7 +378,7 @@ _SECRET_RE = re.compile(
|
||||
_SECRET_VALUE_RE = re.compile(
|
||||
r"-----BEGIN[A-Z ]{0,40}PRIVATE KEY-----[\s\S]*?(?:-----END[A-Z ]{0,40}PRIVATE KEY-----|\Z)"
|
||||
r"|(?:(?<![A-Za-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:gh[pousr]_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255}|(?:AKIA|ASIA)[A-Z0-9]{16})\b"
|
||||
r"|(?:(?<![A-Za-z0-9_-])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:sk-(?:ant|proj|svcacct|admin|or)-[A-Za-z0-9_-]{20,512}|sk-[A-Za-z0-9]{20,512}(?![A-Za-z0-9_-])|eyJ[A-Za-z0-9_-]{8,1024}\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{0,1024})"
|
||||
r"|(?:(?<![A-Za-z0-9_-])|(?<=\\[bfnrt])|(?<=\\u[0-9a-fA-F]{4}))(?:sk-(?:ant|proj|svcacct|admin|or)-(?:[A-Za-z0-9_-]{20,512}|[A-Za-z0-9_-]{0,19}\Z)|sk-(?:[A-Za-z0-9]{20,512}(?![A-Za-z0-9_-])|[A-Za-z0-9]{1,19}\Z)|eyJ[A-Za-z0-9_-]{8,1024}(?:\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{0,1024}|(?:\.[A-Za-z0-9_-]{0,8192})?\Z))"
|
||||
r"""|(?<=://)(?:(?<=https://)|(?<=http://)|(?<!https://)(?<!http://)())([^\s:/@"\\]{1,256}:)(?(1)|(?!\d{1,5}\Z))[^\s@/"\\]{1,256}(?=@|\Z)"""
|
||||
)
|
||||
|
||||
|
||||
@@ -42,6 +42,13 @@ function capture(phase, input, expectedEvent) {
|
||||
}
|
||||
}
|
||||
|
||||
// The persisted input/output is the scrubbed JSON of the tool payload, so one
|
||||
// field can be compared exactly instead of only checking a canary is gone.
|
||||
function capturedField(records, field) {
|
||||
const record = JSON.parse(records);
|
||||
return JSON.parse(record.event === 'tool_start' ? record.input : record.output)[field];
|
||||
}
|
||||
|
||||
console.log('=== Actual registered observation secret redaction ===');
|
||||
if (process.platform === 'win32') {
|
||||
console.log('Skipping shell fixtures: native Windows runner requires platform CI.');
|
||||
@@ -89,6 +96,9 @@ if (process.platform === 'win32') {
|
||||
tool_input: { file_path: '/tmp/example.js', content } });
|
||||
assert.ok(!records.includes(secret), 'seeded credential persisted');
|
||||
assert.ok(records.includes('[REDACTED]'));
|
||||
if (label === 'bare JWT') {
|
||||
assert.equal(capturedField(records, 'content'), content.replace(jwt, '[REDACTED]'));
|
||||
}
|
||||
});
|
||||
test(`redacts ${label} before persisting structured tool output`, () => {
|
||||
const records = capture('post', { tool_name: 'Read',
|
||||
@@ -131,7 +141,8 @@ if (process.platform === 'win32') {
|
||||
tool_response: { stdout: command, stderr: '' } })
|
||||
]) {
|
||||
assert.ok(!records.includes(anthropicCanary), 'Anthropic key persisted');
|
||||
assert.ok(records.includes('[REDACTED]'));
|
||||
const field = JSON.parse(records).event === 'tool_start' ? 'command' : 'stdout';
|
||||
assert.equal(capturedField(records, field), command.replace(anthropicKey, '[REDACTED]'));
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -141,6 +152,32 @@ if (process.platform === 'win32') {
|
||||
assert.ok(records.includes('[REDACTED]'));
|
||||
});
|
||||
|
||||
// Truncation can also cut a bare JWT inside its payload or a key inside its
|
||||
// body. What is left at the end of the capture must still be redacted.
|
||||
const jwtKept = jwt.slice(0, jwt.indexOf('.') + 10);
|
||||
const jwtPayloadStart = jwtKept.slice(jwt.indexOf('.') + 1);
|
||||
const keyKept = anthropicKey.slice(0, 'sk-ant-api03-'.length + 12);
|
||||
const cutAt = (end, value, kept) => `${'a'.repeat(end - kept.length - 1)} ${value}`;
|
||||
for (const [label, value, kept, leak] of [
|
||||
['JWT', jwt, jwtKept, jwtPayloadStart],
|
||||
['Anthropic key', anthropicKey, keyKept, keyKept.slice('sk-ant-api03-'.length)]
|
||||
]) {
|
||||
for (const [where, phase, input, expectedEvent] of [
|
||||
['string tool output', 'post', { tool_name: 'Bash', tool_input: { command: 'env' },
|
||||
tool_response: cutAt(5000, value, kept) }],
|
||||
// json.dumps prefixes '{"content": "' (13 chars) before the value.
|
||||
['structured tool output', 'post', { tool_name: 'Read', tool_input: { file_path: '/tmp/x' },
|
||||
tool_response: { content: cutAt(5000 - 13, value, kept) } }],
|
||||
['parse-error fallback', 'post', JSON.stringify([cutAt(2000 - 2, value, kept)]), 'parse_error']
|
||||
]) {
|
||||
test(`redacts the ${label} cut at the truncation boundary in ${where}`, () => {
|
||||
const records = capture(phase, input, expectedEvent);
|
||||
assert.ok(!records.includes(leak), `cut ${label} persisted`);
|
||||
assert.ok(records.includes('[REDACTED]'));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Capture truncation (5000 chars, 2000 in the parse-error fallback) can cut a
|
||||
// DSN right before its '@'; the password must still be redacted (#3568).
|
||||
const cutSecret = 'CANARYcutPassw0rd';
|
||||
|
||||
Reference in new issue
Block a user