refactor(domain): redact declared sensitive values and transport credentials through one owner (#11327)

* refactor(domain): redact declared sensitive values and transport credentials through one owner

src/domain/explicitSensitiveValues.ts becomes the single owner of how a
declared value (a caller's explicit sensitive value or a REA-minted
transport credential) is found and replaced:

- redactExplicitText gains a preferred replacement marker (still refused
  when it would spell a declaration) and budget cut-off handling, and
  redactExplicitOutput applies cut-off from a stream's observed/retained
  bytes.
- explicitValuesOccurIn and explicitUtf8Spellings replace CaptureRedaction's
  private literal checks; empty declarations no longer match.
- TRANSPORT_CREDENTIAL_HEADERS / isTransportCredentialHeader replace three
  copies of the RFC 9110/6265 header table (capture, Playwright network,
  IDA registration). IDA now also treats Set-Cookie as an authentication
  header.

Migrated: process/ProviderDiagnosticRedaction.ts (deleted; Ghidra client,
diagnostics and launcher, Hopper launcher outcome), AgentEnvComputer's
control-token replaceAll, CaptureRedaction, QLOG, Playwright network
content, IDA registration and diagnostic redactor header checks.

* fix(ida): redact a configured Set-Cookie value without its attributes

* refactor(domain): spell and match declared values through the redaction owner (#11370)

* refactor(domain): spell and match declared values through the redaction owner

* fix(domain): yield declared-value spans lazily so budgeted callers stop early
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 03:17:39 +08:00
1 parent 17e7c26087
commit a97a5b823d
18 files changed
+544 -273

No files matched your search

+22 -27
View File
@@ -5,6 +5,10 @@ import {
} from "./BrowserScenarioSerialization.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import { redactSensitiveFailure } from "../domain/explicitSensitiveFailure.js";
import {
explicitTextSpellings,
lowercasePercentEscapes,
} from "../domain/explicitSensitiveValues.js";
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
import { err, ok, type Result } from "../domain/result.js";
import { createHash } from "node:crypto";
@@ -46,16 +50,6 @@ const normalizedSecretValues = (value: string): readonly string[] => [
...new Set([value.replace(/^[\t ]+|[\t ]+$/gu, ""), value]),
];
const encodedSecretValues = (scalar: string): ReadonlySet<string> => {
const component = encodeURIComponent(scalar);
return new Set([
component,
// Special-URL queries escape apostrophes while keeping the component's tilde.
component.replace(/'/gu, "%27"),
new URLSearchParams([["value", scalar]]).toString().slice("value=".length),
]);
};
type SecretSpellings = "url" | "text";
interface SecretPattern {
readonly text: string;
@@ -72,11 +66,6 @@ interface SecretNeedle {
readonly marker: Buffer;
}
const normalizePercentEscapes = (value: string): string => {
if (!value.includes("%")) return value;
return value.replace(/%[0-9a-f]{2}/giu, (escape) => escape.toLowerCase());
};
const secretPatterns = (
secret: string,
spellings: SecretSpellings,
@@ -92,19 +81,25 @@ const secretPatterns = (
searchText: text,
percentInsensitive: false,
})),
...[...encodedSecretValues(scalar)].map((text) => ({
...explicitTextSpellings(scalar, [
"uri-component",
"uri-query",
"form",
]).map((text) => ({
text,
searchText: normalizePercentEscapes(text),
searchText: lowercasePercentEscapes(text),
percentInsensitive: true,
})),
];
if (spellings === "text")
patterns.push(
...literals.map((text) => ({
text: JSON.stringify(text).slice(1, -1),
searchText: JSON.stringify(text).slice(1, -1),
percentInsensitive: false,
})),
...literals.flatMap((text) =>
explicitTextSpellings(text, ["json-escaped"]).map((escaped) => ({
text: escaped,
searchText: escaped,
percentInsensitive: false,
})),
),
);
const seen = new Set<string>();
return patterns.filter(({ text, percentInsensitive }) => {
@@ -245,7 +240,7 @@ const redactionMarker = (
(pattern) =>
!patternMatches(
candidate,
normalizePercentEscapes(candidate),
lowercasePercentEscapes(candidate),
pattern,
),
),
@@ -262,7 +257,7 @@ const replaceSecretLiterals = (
const normalizedValue = replacements.some(
({ percentInsensitive }) => percentInsensitive,
)
? normalizePercentEscapes(value)
? lowercasePercentEscapes(value)
: value;
const parts: string[] = [];
let offset = 0;
@@ -729,7 +724,7 @@ export class BrowserScenarioSecrets {
);
return containsSecretPattern(
output,
normalizePercentEscapes(output),
lowercasePercentEscapes(output),
patterns,
)
? ""
@@ -817,7 +812,7 @@ export class BrowserScenarioSecrets {
this.urlPatterns,
);
if (
containsSecretPattern(output, normalizePercentEscapes(output), patterns)
containsSecretPattern(output, lowercasePercentEscapes(output), patterns)
)
return {
url: "",
@@ -836,7 +831,7 @@ export class BrowserScenarioSecrets {
const patterns = this.textPatterns;
if (
patterns.length > 0 &&
containsSecretPattern(value, normalizePercentEscapes(value), patterns)
containsSecretPattern(value, lowercasePercentEscapes(value), patterns)
)
return { value_state: "redacted-secret", value_sha256: null };
const hash = createHash("sha256");
+11 -21
View File
@@ -2,26 +2,22 @@ import { escapeJsonPointerToken } from "../../domain/jsonPointer.js";
import { isSafeNumber, type LosslessNumber } from "lossless-json";
import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js";
import { parseRecordedTransportUrl } from "../../domain/recordedTransportUrl.js";
import {
explicitUtf8Spellings,
explicitValuesOccurIn,
isTransportCredentialHeader,
TRANSPORT_CREDENTIAL_HEADERS,
} from "../../domain/explicitSensitiveValues.js";
type Redaction = WebNetworkCaptureRecord["redactions"][number];
type TransportUrlContext = { readonly credentialBaseUrl?: string };
const CREDENTIAL_HEADERS = new Set([
"authorization",
"proxy-authorization",
"cookie",
"set-cookie",
]);
/** HTTP credential names apply equally to recorded headers and gRPC-web trailers. */
export const isCaptureCredentialHeader = (name: string): boolean =>
CREDENTIAL_HEADERS.has(name.toLowerCase());
/** Match an ASCII header name without allocating a potentially enormous line. */
export const isCaptureCredentialHeaderBytes = (
bytes: Uint8Array,
start: number,
end: number,
): boolean => {
for (const name of CREDENTIAL_HEADERS) {
for (const name of TRANSPORT_CREDENTIAL_HEADERS) {
if (end - start !== name.length) continue;
let matches = true;
for (let index = 0; index < name.length; index++) {
@@ -52,12 +48,7 @@ export class CaptureRedaction {
: undefined;
this.#requestUrl =
typeof request?.url === "string" ? request.url : undefined;
// An unpaired surrogate has no UTF-8 representation. Buffer.from would
// invent replacement-character bytes and incorrectly discard real evidence.
this.#sensitiveByteValues = sensitiveValues.flatMap((value) => {
const encoded = Buffer.from(value, "utf8");
return encoded.toString("utf8") === value ? [encoded] : [];
});
this.#sensitiveByteValues = explicitUtf8Spellings(sensitiveValues);
}
/** Preserve exact text unless explicitly sensitive or known transport URL userinfo. */
@@ -76,9 +67,8 @@ export class CaptureRedaction {
}
}
if (
this.sensitiveValues.some(
(literal) => value.includes(literal) || result.includes(literal),
)
explicitValuesOccurIn(value, this.sensitiveValues) ||
explicitValuesOccurIn(result, this.sensitiveValues)
) {
this.redactions.push({ pointer, reason: "explicit-sensitive-value" });
return null;
@@ -116,7 +106,7 @@ export class CaptureRedaction {
return (
(/^\/(request|response)\/headers\/\d+\/value$/.test(pointer) &&
typeof parent.name === "string" &&
isCaptureCredentialHeader(parent.name)) ||
isTransportCredentialHeader(parent.name)) ||
/^\/(request|response)\/cookies\/\d+\/value$/.test(pointer)
);
}
+2 -2
View File
@@ -12,8 +12,8 @@ import {
CaptureRedaction,
captureObject,
capturePointer,
isCaptureCredentialHeader,
} from "./CaptureRedaction.js";
import { isTransportCredentialHeader } from "../../domain/explicitSensitiveValues.js";
import { QLOG_CAPTURE_PROVIDER_IDENTITY } from "./CaptureProviderIdentities.js";
const MAX_VARINT = (1n << 62n) - 1n;
@@ -430,7 +430,7 @@ const project = (
/^\/data\/frame\/headers\/\d+\/value$/u.test(pointer) &&
typeof parent?.name === "string";
if (
(header && isCaptureCredentialHeader(String(parent?.name))) ||
(header && isTransportCredentialHeader(String(parent?.name))) ||
qlogToken(eventName, pointer, parent)
)
return redactor.credential(pointer);
@@ -1,4 +1,5 @@
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import { isTransportCredentialHeader } from "../../domain/explicitSensitiveValues.js";
import {
isJson,
mediaTypeEssence,
@@ -54,13 +55,6 @@ export type NetworkBodyPreflight = (input: {
readonly mediaType: string | undefined;
}) => BrowserScenarioEventAdmission | Promise<BrowserScenarioEventAdmission>;
const credentialHeaders = new Set([
"authorization",
"proxy-authorization",
"cookie",
"set-cookie",
]);
const errorMessage = (
cause: unknown,
secrets: BrowserScenarioSecrets,
@@ -95,7 +89,7 @@ export const captureNetworkHeaders = async (
{ state: "captured" }
>["items"] = [];
for (const { name, value } of headers) {
const credential = credentialHeaders.has(name.toLowerCase());
const credential = isTransportCredentialHeader(name);
cost +=
secrets.redactionAllocationCost(
name.length + (credential ? 0 : value.length),
+4 -1
View File
@@ -22,6 +22,7 @@ import {
} from "./ComputerHomeFiles.js";
import { DRAWN_WAIT_MS, drawnScreenshot } from "./DrawnScreenshot.js";
import { awaitHttpReply, type HttpReply } from "./HttpReply.js";
import { redactExplicitText } from "../domain/explicitSensitiveValues.js";
const READY_TIMEOUT_MS = 180_000;
const REQUEST_TIMEOUT_MS = 120_000;
@@ -697,7 +698,9 @@ export class AgentEnvComputer {
redacted(text: string): string {
return this.#token === null
? text
: text.replaceAll(this.#token, "[control token]");
: redactExplicitText(text, [this.#token], {
replacement: "[control token]",
});
}
/** A framework-only route of the computer, with its control token; its answer is read within `bounds.limit`. */
+20 -37
View File
@@ -3,6 +3,11 @@ import { z } from "zod";
import type { JsonValue } from "../jsonValue.js";
import { gunzip, isGzip } from "./gzipDocument.js";
import { decodeStateText, jsonStringChunks } from "./stateFormats.js";
import {
explicitTextSpellings,
explicitUtf16Spellings,
lowercasePercentEscapes,
} from "../explicitSensitiveValues.js";
/** A secret's handle: the name of one file in the build host's secrets folder. */
export const buildSecretHandleSchema = z
@@ -50,30 +55,13 @@ export const secretValueRefusal = (
return undefined;
};
/** The characters of the base64 encoding of `bytes` that depend on them alone, when they start `offset` bytes into a longer encoding. */
const base64Stable = (bytes: Buffer, offset: number): string => {
const encoded = Buffer.concat([Buffer.alloc(offset), bytes]).toString(
"base64",
);
return encoded.slice(
Math.ceil((8 * offset) / 6),
Math.floor((8 * (offset + bytes.length)) / 6),
);
};
const jsonEscaped = (text: string) => JSON.stringify(text).slice(1, -1);
const jsonEscaped = (text: string): string =>
explicitTextSpellings(text, ["json-escaped"])[0] ?? text;
const xmlEscaped = (text: string) =>
text.replaceAll("&", "&amp;").replaceAll("<", "&lt;");
/** The value's spellings as text: as it is, escaped in XML and in JSON once or twice, in hex, in base64 at each alignment and in URL and form encoding. */
/** The value's spellings as text: as it is, escaped in XML and in JSON once or twice, in hex, in base64 at each alignment and in URL and form encoding with either escape case. */
const textSpellings = (value: string): readonly string[] => {
const bytes = Buffer.from(value, "utf8");
const hex = bytes.toString("hex");
const base64 = [0, 1, 2].map((offset) => base64Stable(bytes, offset));
const url = encodeURIComponent(value);
const form = new URLSearchParams([["value", value]])
.toString()
.slice("value=".length);
const escapedXml = xmlEscaped(value);
const xml = [
value,
@@ -82,33 +70,28 @@ const textSpellings = (value: string): readonly string[] => {
text.replaceAll('"', "&quot;"),
]),
];
const percent = explicitTextSpellings(value, [
"uri-component",
"uri-plus-space",
"form",
]);
return [
...new Set([
...xml,
...xml.map(jsonEscaped),
...xml.map((text) => jsonEscaped(jsonEscaped(text))),
hex,
hex.toUpperCase(),
...base64,
...base64.map((text) => text.replaceAll("+", "-").replaceAll("/", "_")),
url,
url.replaceAll("%20", "+"),
url.replace(/%[0-9A-F]{2}/gu, (escape) => escape.toLowerCase()),
form,
form.replace(/%[0-9A-F]{2}/gu, (escape) => escape.toLowerCase()),
...explicitTextSpellings(value, ["hex", "base64"]),
...percent,
...percent.map(lowercasePercentEscapes),
]),
];
};
/** The byte spellings a scan looks for: each text spelling in UTF-8, and the value in UTF-16 of either byte order. */
const byteSpellings = (value: string): readonly Buffer[] => {
const utf16 = Buffer.from(value, "utf16le");
return [
...textSpellings(value).map((text) => Buffer.from(text, "utf8")),
utf16,
Buffer.from(utf16).swap16(),
];
};
const byteSpellings = (value: string): readonly Buffer[] => [
...textSpellings(value).map((text) => Buffer.from(text, "utf8")),
...explicitUtf16Spellings(value),
];
const holds = (bytes: Uint8Array, spellings: readonly Buffer[]): boolean => {
const haystack = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length);
+153 -6
View File
@@ -1,15 +1,22 @@
import { expect, it } from "vitest";
import { redactExplicitText } from "./explicitSensitiveValues.js";
import {
explicitTextSpellings,
explicitUtf16Spellings,
explicitUtf8Spellings,
explicitValueSpans,
lowercasePercentEscapes,
explicitValuesOccurIn,
isTransportCredentialHeader,
redactExplicitOutput,
redactExplicitText,
} from "./explicitSensitiveValues.js";
it.each([
["secret context", ["secret"], "[REDACTED] context"],
["secret context", ["secret", "REDACTED"], "… context"],
["secret context", ["secret", "[", "…"], " context"],
[
"secretable context",
["secret", "able", "table"],
"[REDACTED][REDACTED] context",
],
// Overlapping declarations redact as one span, leaving no fragment between.
["secretable context", ["secret", "able", "table"], "[REDACTED] context"],
["secretsecret context", ["secret", "…", "REDACTED"], " context"],
] as const)(
"keeps declared literals out of replacement text: %s",
@@ -24,4 +31,144 @@ it("does not guess sensitivity or treat empty declarations as matches", () => {
expect(redactExplicitText("API_SECRET ordinary evidence", [""])).toBe(
"API_SECRET ordinary evidence",
);
expect(explicitValuesOccurIn("API_SECRET ordinary evidence", [""])).toBe(
false,
);
});
it("uses a caller's marker unless it would spell a declaration", () => {
expect(
redactExplicitText("token=abc", ["abc"], {
replacement: "[control token]",
}),
).toBe("token=[control token]");
expect(
redactExplicitText("token=con", ["con"], {
replacement: "[control token]",
}),
).toBe("token=[REDACTED]");
});
it.each([
[
"complete output keeps an unrelated tail",
false,
"log private",
"log private",
],
[
"cut output replaces a declared prefix",
true,
"log private-tr",
"log [REDACTED]",
],
[
"cut output replaces whole values before the tail",
true,
"private-transport-token then private-",
"[REDACTED] then [REDACTED]",
],
[
"cut output without a declared prefix is unchanged",
true,
"log done",
"log done",
],
] as const)("%s", (_name, cutOff, text, expected) => {
expect(
redactExplicitText(text, ["private-transport-token"], { cutOff }),
).toBe(expected);
});
it("treats output as cut when more bytes were observed than retained", () => {
const token = "private-transport-token";
expect(
redactExplicitOutput(
{ text: token.slice(0, 8), bytes: 8, observedBytes: token.length },
[token],
),
).toBe("[REDACTED]");
expect(
redactExplicitOutput({ text: "priv", bytes: 4, observedBytes: 4 }, [token]),
).toBe("priv");
});
it("spells declarations in UTF-8 and skips values with no UTF-8 form", () => {
expect(explicitUtf8Spellings(["é", "\ud800", "", "é"])).toEqual([
Buffer.from("é", "utf8"),
]);
});
it("recognizes protocol-defined credential headers in any case only", () => {
for (const name of [
"Authorization",
"proxy-authorization",
"Cookie",
"SET-COOKIE",
])
expect(isTransportCredentialHeader(name)).toBe(true);
for (const name of ["x-api-key", "x-auth-token", "location"])
expect(isTransportCredentialHeader(name)).toBe(false);
});
it("merges overlapping occurrences of one or several declarations", () => {
expect([...explicitValueSpans("aaaa", ["aa"])]).toEqual([
{ start: 0, end: 4 },
]);
expect([
...explicitValueSpans("abcdef xx cd", ["abcd", "cdef", "cd"]),
]).toEqual([
{ start: 0, end: 6 },
{ start: 10, end: 12 },
]);
expect([...explicitValueSpans("abab", ["ab"])]).toEqual([
{ start: 0, end: 2 },
{ start: 2, end: 4 },
]);
expect([...explicitValueSpans("text", [""])]).toEqual([]);
});
it("spells a declared value in each selected encoding", () => {
expect(
explicitTextSpellings("a b'/é", [
"uri-component",
"uri-query",
"uri-plus-space",
"form",
]),
).toEqual([
"a%20b'%2F%C3%A9",
"a%20b%27%2F%C3%A9",
"a+b'%2F%C3%A9",
"a+b%27%2F%C3%A9",
]);
expect(explicitTextSpellings('q"\\n', ["json-escaped"])).toEqual([
'q\\"\\\\n',
]);
expect(explicitTextSpellings("ok", ["json-escaped"])).toEqual([]);
expect(explicitTextSpellings("\u00ff", ["hex"])).toEqual(["c3bf", "C3BF"]);
const base64 = explicitTextSpellings("secret-value", ["base64"]);
for (const prefix of ["", "x", "xy"])
expect(
base64.some((spelling) =>
Buffer.from(`${prefix}secret-value`)
.toString("base64")
.includes(spelling),
),
).toBe(true);
// URL serializers write an unpaired surrogate as U+FFFD.
expect(explicitTextSpellings("\ud800", ["uri-component"])).toEqual([
"%EF%BF%BD",
]);
expect(explicitUtf16Spellings("ab")).toEqual([
Buffer.from([0x61, 0, 0x62, 0]),
Buffer.from([0, 0x61, 0, 0x62]),
]);
expect(lowercasePercentEscapes("a%2Fb%C3")).toBe("a%2fb%c3");
});
it("finds spans lazily, so a budgeted caller stops matching early", () => {
const spans = explicitValueSpans("token ".repeat(100_000), ["token"]);
expect(spans.next().value).toEqual({ start: 0, end: 5 });
expect(spans.next().value).toEqual({ start: 6, end: 11 });
});
+250 -10
View File
@@ -1,20 +1,260 @@
/** Replace caller-declared literal text values without guessing sensitivity from names. */
import { toUSVString } from "node:util";
/**
* The one owner for redacting declared sensitive values: literals a caller
* explicitly marked sensitive, and transport credentials REA minted itself.
* Nothing here infers secrecy from names or text patterns; an undeclared value
* is ordinary evidence. Producers choose what they declare, this module
* decides how every declared value is found and replaced.
*/
/**
* HTTP fields whose values are transport credentials by protocol definition
* (RFC 9110 authentication, RFC 6265 cookies), in lower case. Other headers
* are ordinary evidence unless a caller declares them.
*/
export const TRANSPORT_CREDENTIAL_HEADERS: ReadonlySet<string> = new Set([
"authorization",
"proxy-authorization",
"cookie",
"set-cookie",
]);
/** Whether an HTTP field name carries a transport credential. */
export const isTransportCredentialHeader = (name: string): boolean =>
TRANSPORT_CREDENTIAL_HEADERS.has(name.toLowerCase());
/** Distinct non-empty declarations, longest first so overlaps redact whole. */
const declaredLiterals = (values: readonly string[]): readonly string[] =>
[...new Set(values)]
.filter((literal) => literal !== "")
.sort((left, right) => right.length - left.length);
/** Whether any declared literal occurs in `value`. */
export const explicitValuesOccurIn = (
value: string,
sensitiveValues: readonly string[],
): boolean =>
sensitiveValues.some((literal) => literal !== "" && value.includes(literal));
/**
* UTF-8 spellings of the declared literals, for searching retained bytes. An
* unpaired surrogate has no UTF-8 representation; encoding it would invent
* replacement-character bytes and discard real evidence, so it is skipped.
*/
export const explicitUtf8Spellings = (
sensitiveValues: readonly string[],
): readonly Buffer[] =>
declaredLiterals(sensitiveValues).flatMap((value) => {
const encoded = Buffer.from(value, "utf8");
return encoded.toString("utf8") === value ? [encoded] : [];
});
/** One occurrence of declared text, as a half-open UTF-16 range. */
export type ExplicitValueSpan = Readonly<{ start: number; end: number }>;
/**
* Every occurrence of any declared literal in `value`, in order. Overlapping
* occurrences (of one literal or of several) merge into one span, so no part
* of an overlapping declaration survives between two replacements. Spans are
* produced lazily with one cursor per literal, so a caller that stops at a
* budget does no further matching work.
*/
export function* explicitValueSpans(
value: string,
sensitiveValues: readonly string[],
): Generator<ExplicitValueSpan, void, undefined> {
const literals = declaredLiterals(sensitiveValues);
const next = literals.map((literal) => value.indexOf(literal));
let pending: ExplicitValueSpan | undefined;
for (;;) {
// Earliest occurrence; at one start, the longest literal (listed first).
let selected = -1;
for (let index = 0; index < literals.length; index += 1)
if (next[index]! >= 0 && (selected < 0 || next[index]! < next[selected]!))
selected = index;
if (selected < 0) break;
const start = next[selected]!;
const end = start + literals[selected]!.length;
next[selected] = value.indexOf(literals[selected]!, start + 1);
if (pending !== undefined && start < pending.end)
pending = { start: pending.start, end: Math.max(pending.end, end) };
else {
if (pending !== undefined) yield pending;
pending = { start, end };
}
}
if (pending !== undefined) yield pending;
}
/**
* Encoded spellings a declared value can take in retained evidence:
* - `uri-component`: `encodeURIComponent`;
* - `uri-query`: a WHATWG special-URL query, which also escapes `'`;
* - `uri-plus-space`: a URI component whose spaces are written `+`;
* - `form`: `application/x-www-form-urlencoded`;
* - `json-escaped`: the body of a JSON string literal;
* - `hex`: UTF-8 bytes in lower- and upper-case hex;
* - `base64`: UTF-8 bytes in base64 and base64url at each of the three byte
* alignments inside a longer encoding (only the characters that depend on
* the value alone).
* Percent escapes are upper case; see {@link lowercasePercentEscapes}.
*/
export type ExplicitSpelling =
| "uri-component"
| "uri-query"
| "uri-plus-space"
| "form"
| "json-escaped"
| "hex"
| "base64";
/** Characters of `bytes`' base64 that depend on them alone at a byte offset. */
const base64Stable = (bytes: Buffer, offset: number): string => {
const encoded = Buffer.concat([Buffer.alloc(offset), bytes]).toString(
"base64",
);
return encoded.slice(
Math.ceil((8 * offset) / 6),
Math.floor((8 * (offset + bytes.length)) / 6),
);
};
const spellingsOf = (value: string, spelling: ExplicitSpelling): string[] => {
// URL and form serializers replace unpaired surrogates with U+FFFD.
const scalar = toUSVString(value);
switch (spelling) {
case "uri-component":
return [encodeURIComponent(scalar)];
case "uri-query":
return [encodeURIComponent(scalar).replace(/'/gu, "%27")];
case "uri-plus-space":
return [encodeURIComponent(scalar).replaceAll("%20", "+")];
case "form":
return [
new URLSearchParams([["value", scalar]])
.toString()
.slice("value=".length),
];
case "json-escaped":
return [JSON.stringify(value).slice(1, -1)];
case "hex": {
const hex = Buffer.from(value, "utf8").toString("hex");
return [hex, hex.toUpperCase()];
}
case "base64": {
const bytes = Buffer.from(value, "utf8");
const aligned = [0, 1, 2].map((offset) => base64Stable(bytes, offset));
return [
...aligned,
...aligned.map((text) =>
text.replaceAll("+", "-").replaceAll("/", "_"),
),
];
}
}
};
/**
* The selected encoded spellings of one declared value, distinct and
* non-empty, excluding the value itself.
*/
export const explicitTextSpellings = (
value: string,
spellings: readonly ExplicitSpelling[],
): readonly string[] => [
...new Set(
spellings
.flatMap((spelling) => spellingsOf(value, spelling))
.filter((text) => text !== "" && text !== value),
),
];
/** The value in UTF-16 of both byte orders, for searching retained bytes. */
export const explicitUtf16Spellings = (value: string): readonly Buffer[] => {
const little = Buffer.from(value, "utf16le");
return [little, Buffer.from(little).swap16()];
};
/** Lower-case every percent escape, so spellings compare escape-case-insensitively. */
export const lowercasePercentEscapes = (value: string): string =>
value.includes("%")
? value.replace(/%[0-9a-f]{2}/giu, (escape) => escape.toLowerCase())
: value;
export type ExplicitRedactionOptions = Readonly<{
/** Preferred marker; a safe fallback is used when it contains a declaration. */
replacement?: string;
/**
* The text was cut by an output budget, so its end may hold the start of a
* declared value. The longest such prefix is replaced as well.
*/
cutOff?: boolean;
}>;
/** Replace declared literal text values without guessing sensitivity from names. */
export const redactExplicitText = (
value: string,
sensitiveValues: readonly string[],
options: ExplicitRedactionOptions = {},
): string => {
const literals = [...new Set(sensitiveValues)].filter(
(literal) => literal !== "",
);
const literals = declaredLiterals(sensitiveValues);
if (literals.length === 0) return value;
const replacement =
["[REDACTED]", "…", ""].find((candidate) =>
[
...(options.replacement === undefined ? [] : [options.replacement]),
"[REDACTED]",
"…",
"",
].find((candidate) =>
literals.every((literal) => !candidate.includes(literal)),
) ?? "";
let result = value;
for (const literal of literals.sort(
(left, right) => right.length - left.length,
))
result = result.replaceAll(literal, replacement);
let result = "";
let cursor = 0;
for (const { start, end } of explicitValueSpans(value, literals)) {
result += value.slice(cursor, start) + replacement;
cursor = end;
}
result += value.slice(cursor);
if (options.cutOff === true) {
const tail = cutOffPrefixLength(result, literals);
if (tail > 0)
result = `${result.slice(0, result.length - tail)}${replacement}`;
}
// Replacement boundaries can form another declaration; exclude that text.
return literals.some((literal) => result.includes(literal)) ? "" : result;
};
/**
* Redact declared values in retained output. Output whose observed length
* exceeds the retained bytes was cut by its budget, so its end is checked for
* the start of a declared value too.
*/
export const redactExplicitOutput = (
output: Readonly<{ text: string; bytes: number; observedBytes: number }>,
sensitiveValues: readonly string[],
replacement?: string,
): string =>
redactExplicitText(output.text, sensitiveValues, {
cutOff: output.observedBytes > output.bytes,
...(replacement === undefined ? {} : { replacement }),
});
/** Length of the longest proper prefix of a declaration that ends `text`. */
const cutOffPrefixLength = (
text: string,
literals: readonly string[],
): number => {
let longest = 0;
for (const literal of literals)
for (
let length = Math.min(text.length, literal.length - 1);
length > longest;
length -= 1
)
if (text.endsWith(literal.slice(0, length))) {
longest = length;
break;
}
return longest;
};
+21 -34
View File
@@ -37,6 +37,7 @@ import {
import { AnalysisError } from "../domain/analysisErrorBase.js";
import { isValidFridaRemoteAddress } from "../domain/fridaRemoteAddress.js";
import { redactExplicitFailure } from "../domain/explicitSensitiveFailure.js";
import { explicitValueSpans } from "../domain/explicitSensitiveValues.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js";
import type { JsonValue } from "../domain/jsonValue.js";
@@ -1392,59 +1393,45 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort {
}
}
/** Redact declared values, then keep what fits `maximumBytes` of JSON string body. */
const boundedRedactedString = (
input: string,
maximumBytes: number,
secrets: readonly string[],
): { readonly value: string; readonly truncated: boolean } => {
const replacement = "[redacted]";
const replacementBytes = jsonStringByteLength(replacement) - 2;
const chunks: string[] = [];
let outputBytes = 0;
let cursor = 0;
let segmentStart = 0;
let truncated = false;
while (cursor < input.length) {
let nextSecretStart = input.length;
let nextSecret: string | undefined;
for (const secret of secrets) {
const match = input.indexOf(secret, cursor);
if (match !== -1 && match < nextSecretStart) {
nextSecretStart = match;
nextSecret = secret;
}
}
if (nextSecretStart === cursor && nextSecret !== undefined) {
const replacement = "[redacted]";
const replacementBytes = jsonStringByteLength(replacement) - 2;
if (outputBytes + replacementBytes > maximumBytes) {
truncated = true;
break;
}
chunks.push(input.slice(segmentStart, cursor), replacement);
outputBytes += replacementBytes;
cursor += nextSecret.length;
segmentStart = cursor;
continue;
}
while (cursor < nextSecretStart) {
const keep = (end: number): boolean => {
const start = cursor;
while (cursor < end) {
const codePoint = input.codePointAt(cursor);
if (codePoint === undefined) break;
const character = String.fromCodePoint(codePoint);
const characterBytes = jsonStringByteLength(character) - 2;
if (
cursor + character.length > nextSecretStart ||
cursor + character.length > end ||
outputBytes + characterBytes > maximumBytes
) {
truncated = true;
)
break;
}
outputBytes += characterBytes;
cursor += character.length;
}
if (truncated) break;
chunks.push(input.slice(start, cursor));
return cursor === end;
};
for (const span of explicitValueSpans(input, secrets)) {
if (!keep(span.start)) return { value: chunks.join(""), truncated: true };
if (outputBytes + replacementBytes > maximumBytes)
return { value: chunks.join(""), truncated: true };
chunks.push(replacement);
outputBytes += replacementBytes;
cursor = span.end;
}
chunks.push(input.slice(segmentStart, cursor));
if (cursor < input.length) truncated = true;
return { value: chunks.join(""), truncated };
const complete = keep(input.length);
return { value: chunks.join(""), truncated: !complete };
};
type BoundedJson = {
+4 -10
View File
@@ -104,7 +104,7 @@ import {
RegularFileChangedError,
sameRegularFileState,
} from "../filesystem/RegularFile.js";
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
import { redactExplicitText } from "../domain/explicitSensitiveValues.js";
export type GhidraTargetSnapshot =
| { readonly kind: "captured"; readonly bytes: Buffer }
@@ -1537,15 +1537,9 @@ export class GhidraClient {
}
#redactAuthentication(value: string, incomplete = false): string {
for (const token of this.#authenticationTokens) {
const bytes = Buffer.byteLength(value);
value = redactCapturedTransportCredential(
{ text: value, bytes, observedBytes: bytes + (incomplete ? 1 : 0) },
token,
"[REDACTED]",
);
}
return value;
return redactExplicitText(value, [...this.#authenticationTokens], {
cutOff: incomplete,
});
}
#onProcessDiagnostic(event: ProviderProcessDiagnostic): void {
+12 -14
View File
@@ -1,6 +1,9 @@
import type { JsonValue } from "../domain/jsonValue.js";
import { utf8TailExcerpt } from "../domain/utf8Excerpt.js";
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
import {
redactExplicitOutput,
redactExplicitText,
} from "../domain/explicitSensitiveValues.js";
import type {
ProviderProcessOutputSince,
ProviderProcessSnapshot,
@@ -141,23 +144,19 @@ export const ghidraFailureOutput = (
complete: scoped.complete && !drainIncomplete,
launcher_failures: observedLauncherFailures(scoped, token),
observed_oome: (scoped.markers ?? []).map((marker) => {
const bytes = Buffer.byteLength(marker.line, "utf8");
// The supervisor's runtime owns which memory area the line names.
const { area } = runtimeExhaustion("jvm", marker.stream, marker);
return {
stream: marker.stream,
offset: marker.offset,
...(area === undefined ? {} : { area }),
line: redactCapturedTransportCredential(
// A cut line may end in a partial credential.
line: redactExplicitText(
marker.line,
token === undefined ? [] : [token],
{
text: marker.line,
bytes,
// A cut line may end in a partial credential.
observedBytes:
marker.trailingContextIncomplete === true ? bytes + 1 : bytes,
cutOff: marker.trailingContextIncomplete === true,
},
token,
"[REDACTED]",
),
line_complete: marker.trailingContextIncomplete !== true,
...(marker.truncated ? { truncated: true } : {}),
@@ -240,10 +239,9 @@ const observedLauncherFailures = (
const failures: GhidraLauncherFailure[] = [];
let truncated = false;
for (const stream of ["stdout", "stderr"] as const) {
const text = redactCapturedTransportCredential(
const text = redactExplicitOutput(
scoped.snapshot[stream],
token,
"[REDACTED]",
token === undefined ? [] : [token],
);
for (const line of text.split(/\r?\n/u)) {
const failure = ghidraLauncherFailure(stream, line);
@@ -267,7 +265,7 @@ export const ghidraDiagnosticStream = (
): JsonValue => {
// Redact the whole retained text first, so the cut cannot expose part of a token.
const excerpt = utf8TailExcerpt(
redactCapturedTransportCredential(stream, token, "[REDACTED]"),
redactExplicitOutput(stream, token === undefined ? [] : [token]),
GHIDRA_DIAGNOSTIC_EXCERPT_BYTES,
);
return {
+3 -8
View File
@@ -48,7 +48,7 @@ import {
execFileFailureCapturedOutput,
} from "../process/ExecFileOutput.js";
import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js";
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
import { redactExplicitText } from "../domain/explicitSensitiveValues.js";
import {
cleanupOwnedProcessGroup,
type ProcessCleanupResult,
@@ -517,12 +517,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher {
environment,
onStderr: (mode, stderr) => {
if (stderr === "") return;
const bytes = Buffer.byteLength(stderr);
const redacted = redactCapturedTransportCredential(
{ text: stderr, bytes, observedBytes: bytes },
session.token,
"[REDACTED]",
);
const redacted = redactExplicitText(stderr, [session.token]);
const bounded = utf8HeadExcerpt(
redacted,
GHIDRA_LAUNCH_COMMAND_OUTPUT_BYTES,
@@ -530,7 +525,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher {
launchSupport.push({
mode,
stderr: bounded.text,
stderr_bytes: bytes,
stderr_bytes: Buffer.byteLength(stderr),
stderr_truncated: bounded.truncated,
});
},
+5 -5
View File
@@ -61,7 +61,7 @@ import {
ProviderStartupDeadline,
waitForAbortableDelay,
} from "../process/ProviderDeadline.js";
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
import { redactExplicitOutput } from "../domain/explicitSensitiveValues.js";
import { ProviderRunLineage } from "../process/ProviderRunLineage.js";
import { AbortedWait, waitForAbortable } from "../process/AbortablePromise.js";
import { SharedStartupAttempt } from "../process/SharedStartupAttempt.js";
@@ -1116,18 +1116,18 @@ export class HopperClient<
exit_code: snapshot.exitCode ?? null,
signal: snapshot.signal ?? null,
stdout: {
text: redactCapturedTransportCredential(
text: redactExplicitOutput(
snapshot.stdout,
token,
token === undefined ? [] : [token],
"[redacted transport credential]",
),
bytes: snapshot.stdout.observedBytes,
retained_bytes: snapshot.stdout.bytes,
},
stderr: {
text: redactCapturedTransportCredential(
text: redactExplicitOutput(
snapshot.stderr,
token,
token === undefined ? [] : [token],
"[redacted transport credential]",
),
bytes: snapshot.stderr.observedBytes,
+12
View File
@@ -80,6 +80,18 @@ describe("IDA diagnostic credential redaction", () => {
"raw=[redacted transport credential]; wire=[redacted transport credential]; token=[redacted transport credential]; proxy=[redacted transport credential]; bare=[redacted transport credential]; cookie=[redacted transport credential]",
);
});
it("redacts a Set-Cookie value but not its attributes", () => {
const redact = createIdaDiagnosticRedactor({
url: "http://127.0.0.1:12345/mcp",
headers: { "Set-Cookie": "sid=private-secret; Path=/; HttpOnly" },
mode: "attached",
protocol: "legacy",
timeoutMs: 1000,
}).text;
expect(redact("rejected private-secret at /caller/input")).toBe(
"rejected [redacted transport credential] at /caller/input",
);
});
it("does not treat empty HTTP authentication as diagnostic content", () => {
const redact = createIdaDiagnosticRedactor({
url: "http://127.0.0.1:12345/mcp",
+17 -44
View File
@@ -9,10 +9,11 @@ import {
writeIdaDiagnostic,
type IdaDiagnosticMatch,
} from "./IdaDiagnosticWriter.js";
import { idaExplicitSensitiveValues } from "./IdaRegistrationIdentity.js";
import {
isIdaAuthenticationHeader,
idaExplicitSensitiveValues,
} from "./IdaRegistrationIdentity.js";
explicitValueSpans,
isTransportCredentialHeader,
} from "../domain/explicitSensitiveValues.js";
import type { IdaConfiguration } from "./IdaConfiguration.js";
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
@@ -155,58 +156,30 @@ export const createIdaDiagnosticRedactor = (
? Object.entries(config.headers)
: [];
const values = headers.flatMap(([name, value]) => {
if (!isIdaAuthenticationHeader(name)) return [];
if (!isTransportCredentialHeader(name)) return [];
// Fetch normalizes surrounding HTTP whitespace before sending headers.
const normalized = value.replace(/^[ \t\r\n]+|[ \t\r\n]+$/gu, "");
if (normalized.length === 0) return [];
const field = name.toLowerCase();
return [
value,
normalized,
...(name.toLowerCase() === "cookie"
...(field === "cookie"
? cookiePayloads(normalized)
: authorizationPayloads(normalized)),
: field === "set-cookie"
? // RFC 6265 4.1: only the leading cookie-pair carries the credential;
// attributes such as Path=/ are not secrets.
cookiePayloads(normalized.split(";", 1)[0] ?? "")
: authorizationPayloads(normalized)),
];
});
const secrets = [
...new Set(
[...values, ...idaExplicitSensitiveValues(config)].filter(
(value) => value.length > 0,
),
),
].sort((left, right) => right.length - left.length);
const secrets = [...values, ...idaExplicitSensitiveValues(config)].filter(
(value) => value.length > 0,
);
const replacement =
secrets.length === 0 ? IDA_REDACTION_MARKER : redactionMarker(secrets);
const alternatives = secrets
.map((secret) => secret.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"))
.join("|");
// Look ahead at each source position; a global consuming match loses overlap.
// Longest-first alternatives cover every shorter match at the same position.
const pattern =
alternatives.length === 0
? undefined
: new RegExp("(?=(" + alternatives + "))", "gu");
const matches = function* (value: string): Iterable<IdaDiagnosticMatch> {
if (pattern === undefined) return;
let pending: IdaDiagnosticMatch | undefined;
for (const match of value.matchAll(pattern)) {
const credential = match[1];
if (credential === undefined || credential.length === 0)
throw new AnalysisProtocolError(
"IDA diagnostic matching omitted its credential span.",
);
const span = { start: match.index, end: match.index + credential.length };
if (pending !== undefined && span.start < pending.end) {
pending = {
start: pending.start,
end: Math.max(pending.end, span.end),
};
} else {
if (pending !== undefined) yield pending;
pending = span;
}
}
if (pending !== undefined) yield pending;
};
const matches = (value: string): Iterable<IdaDiagnosticMatch> =>
explicitValueSpans(value, secrets);
const text = boundedDiagnostic((message) =>
writeIdaDiagnostic(message, matches(message), replacement),
);
+2 -7
View File
@@ -1,14 +1,9 @@
import { digestCanonicalValue } from "../domain/canonicalDigest.js";
import { isTransportCredentialHeader } from "../domain/explicitSensitiveValues.js";
import type { IdaConfiguration } from "./IdaConfiguration.js";
/** HTTP authentication fields have protocol-defined credential meaning. */
export const isIdaAuthenticationHeader = (name: string): boolean =>
["authorization", "proxy-authorization", "cookie"].includes(
name.toLowerCase(),
);
const isSensitiveHeader = (config: IdaConfiguration, name: string): boolean =>
isIdaAuthenticationHeader(name) ||
isTransportCredentialHeader(name) ||
(config.sensitive?.headers.some(
(selected) => selected.toLowerCase() === name.toLowerCase(),
) ??
@@ -1,31 +0,0 @@
import type { ProviderProcessSnapshot } from "./ProviderProcess.js";
/** Redact explicit transport credentials from complete and budget-cut output. */
export const redactCapturedTransportCredential = (
stream: Pick<
ProviderProcessSnapshot["stdout"],
"text" | "bytes" | "observedBytes"
>,
token: string | undefined,
replacement: string,
): string => {
if (token === undefined || token.length === 0) return stream.text;
const { text } = stream;
const cutOff = stream.observedBytes > stream.bytes;
if (cutOff && !text.endsWith(token)) {
const maximumPrefixLength = Math.min(text.length, token.length - 1);
for (
let prefixLength = maximumPrefixLength;
prefixLength > 0;
prefixLength -= 1
) {
if (text.endsWith(token.slice(0, prefixLength)))
return `${text
.slice(0, text.length - prefixLength)
.replaceAll(token, replacement)}${replacement}`;
}
}
return text.replaceAll(token, replacement);
};
+4 -8
View File
@@ -7,7 +7,7 @@ import {
ProviderProcessSupervisor,
type ProcessDiagnosticBudget,
} from "./ProviderProcess.js";
import { redactCapturedTransportCredential } from "./ProviderDiagnosticRedaction.js";
import { redactExplicitOutput } from "../domain/explicitSensitiveValues.js";
it("retains final stderr between verified exit and output close during stop", async () => {
const producer = Object.assign(new EventEmitter(), {
@@ -544,13 +544,9 @@ it.each(["stdout", "stderr"] as const)(
observedBytes: Buffer.byteLength(token),
text: token.slice(0, 8),
});
expect(
redactCapturedTransportCredential(
scoped.snapshot[stream],
token,
"[REDACTED]",
),
).toBe("[REDACTED]");
expect(redactExplicitOutput(scoped.snapshot[stream], [token])).toBe(
"[REDACTED]",
);
} finally {
supervisor.dispose();
producer.stdout.destroy();