mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 11:43:20 +00:00
refactor(domain): redact declared sensitive values and transport credentials through one owner (#11327)
* refactor(domain): redact declared sensitive values and transport credentials through one owner src/domain/explicitSensitiveValues.ts becomes the single owner of how a declared value (a caller's explicit sensitive value or a REA-minted transport credential) is found and replaced: - redactExplicitText gains a preferred replacement marker (still refused when it would spell a declaration) and budget cut-off handling, and redactExplicitOutput applies cut-off from a stream's observed/retained bytes. - explicitValuesOccurIn and explicitUtf8Spellings replace CaptureRedaction's private literal checks; empty declarations no longer match. - TRANSPORT_CREDENTIAL_HEADERS / isTransportCredentialHeader replace three copies of the RFC 9110/6265 header table (capture, Playwright network, IDA registration). IDA now also treats Set-Cookie as an authentication header. Migrated: process/ProviderDiagnosticRedaction.ts (deleted; Ghidra client, diagnostics and launcher, Hopper launcher outcome), AgentEnvComputer's control-token replaceAll, CaptureRedaction, QLOG, Playwright network content, IDA registration and diagnostic redactor header checks. * fix(ida): redact a configured Set-Cookie value without its attributes * refactor(domain): spell and match declared values through the redaction owner (#11370) * refactor(domain): spell and match declared values through the redaction owner * fix(domain): yield declared-value spans lazily so budgeted callers stop early
This commit is contained in:
1 parent
17e7c26087
commit
a97a5b823d
18 files changed
+544
-273
No files matched your search
@@ -5,6 +5,10 @@ import {
|
||||
} from "./BrowserScenarioSerialization.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { redactSensitiveFailure } from "../domain/explicitSensitiveFailure.js";
|
||||
import {
|
||||
explicitTextSpellings,
|
||||
lowercasePercentEscapes,
|
||||
} from "../domain/explicitSensitiveValues.js";
|
||||
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -46,16 +50,6 @@ const normalizedSecretValues = (value: string): readonly string[] => [
|
||||
...new Set([value.replace(/^[\t ]+|[\t ]+$/gu, ""), value]),
|
||||
];
|
||||
|
||||
const encodedSecretValues = (scalar: string): ReadonlySet<string> => {
|
||||
const component = encodeURIComponent(scalar);
|
||||
return new Set([
|
||||
component,
|
||||
// Special-URL queries escape apostrophes while keeping the component's tilde.
|
||||
component.replace(/'/gu, "%27"),
|
||||
new URLSearchParams([["value", scalar]]).toString().slice("value=".length),
|
||||
]);
|
||||
};
|
||||
|
||||
type SecretSpellings = "url" | "text";
|
||||
interface SecretPattern {
|
||||
readonly text: string;
|
||||
@@ -72,11 +66,6 @@ interface SecretNeedle {
|
||||
readonly marker: Buffer;
|
||||
}
|
||||
|
||||
const normalizePercentEscapes = (value: string): string => {
|
||||
if (!value.includes("%")) return value;
|
||||
return value.replace(/%[0-9a-f]{2}/giu, (escape) => escape.toLowerCase());
|
||||
};
|
||||
|
||||
const secretPatterns = (
|
||||
secret: string,
|
||||
spellings: SecretSpellings,
|
||||
@@ -92,19 +81,25 @@ const secretPatterns = (
|
||||
searchText: text,
|
||||
percentInsensitive: false,
|
||||
})),
|
||||
...[...encodedSecretValues(scalar)].map((text) => ({
|
||||
...explicitTextSpellings(scalar, [
|
||||
"uri-component",
|
||||
"uri-query",
|
||||
"form",
|
||||
]).map((text) => ({
|
||||
text,
|
||||
searchText: normalizePercentEscapes(text),
|
||||
searchText: lowercasePercentEscapes(text),
|
||||
percentInsensitive: true,
|
||||
})),
|
||||
];
|
||||
if (spellings === "text")
|
||||
patterns.push(
|
||||
...literals.map((text) => ({
|
||||
text: JSON.stringify(text).slice(1, -1),
|
||||
searchText: JSON.stringify(text).slice(1, -1),
|
||||
percentInsensitive: false,
|
||||
})),
|
||||
...literals.flatMap((text) =>
|
||||
explicitTextSpellings(text, ["json-escaped"]).map((escaped) => ({
|
||||
text: escaped,
|
||||
searchText: escaped,
|
||||
percentInsensitive: false,
|
||||
})),
|
||||
),
|
||||
);
|
||||
const seen = new Set<string>();
|
||||
return patterns.filter(({ text, percentInsensitive }) => {
|
||||
@@ -245,7 +240,7 @@ const redactionMarker = (
|
||||
(pattern) =>
|
||||
!patternMatches(
|
||||
candidate,
|
||||
normalizePercentEscapes(candidate),
|
||||
lowercasePercentEscapes(candidate),
|
||||
pattern,
|
||||
),
|
||||
),
|
||||
@@ -262,7 +257,7 @@ const replaceSecretLiterals = (
|
||||
const normalizedValue = replacements.some(
|
||||
({ percentInsensitive }) => percentInsensitive,
|
||||
)
|
||||
? normalizePercentEscapes(value)
|
||||
? lowercasePercentEscapes(value)
|
||||
: value;
|
||||
const parts: string[] = [];
|
||||
let offset = 0;
|
||||
@@ -729,7 +724,7 @@ export class BrowserScenarioSecrets {
|
||||
);
|
||||
return containsSecretPattern(
|
||||
output,
|
||||
normalizePercentEscapes(output),
|
||||
lowercasePercentEscapes(output),
|
||||
patterns,
|
||||
)
|
||||
? ""
|
||||
@@ -817,7 +812,7 @@ export class BrowserScenarioSecrets {
|
||||
this.urlPatterns,
|
||||
);
|
||||
if (
|
||||
containsSecretPattern(output, normalizePercentEscapes(output), patterns)
|
||||
containsSecretPattern(output, lowercasePercentEscapes(output), patterns)
|
||||
)
|
||||
return {
|
||||
url: "",
|
||||
@@ -836,7 +831,7 @@ export class BrowserScenarioSecrets {
|
||||
const patterns = this.textPatterns;
|
||||
if (
|
||||
patterns.length > 0 &&
|
||||
containsSecretPattern(value, normalizePercentEscapes(value), patterns)
|
||||
containsSecretPattern(value, lowercasePercentEscapes(value), patterns)
|
||||
)
|
||||
return { value_state: "redacted-secret", value_sha256: null };
|
||||
const hash = createHash("sha256");
|
||||
|
||||
@@ -2,26 +2,22 @@ import { escapeJsonPointerToken } from "../../domain/jsonPointer.js";
|
||||
import { isSafeNumber, type LosslessNumber } from "lossless-json";
|
||||
import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js";
|
||||
import { parseRecordedTransportUrl } from "../../domain/recordedTransportUrl.js";
|
||||
import {
|
||||
explicitUtf8Spellings,
|
||||
explicitValuesOccurIn,
|
||||
isTransportCredentialHeader,
|
||||
TRANSPORT_CREDENTIAL_HEADERS,
|
||||
} from "../../domain/explicitSensitiveValues.js";
|
||||
|
||||
type Redaction = WebNetworkCaptureRecord["redactions"][number];
|
||||
type TransportUrlContext = { readonly credentialBaseUrl?: string };
|
||||
const CREDENTIAL_HEADERS = new Set([
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"cookie",
|
||||
"set-cookie",
|
||||
]);
|
||||
/** HTTP credential names apply equally to recorded headers and gRPC-web trailers. */
|
||||
export const isCaptureCredentialHeader = (name: string): boolean =>
|
||||
CREDENTIAL_HEADERS.has(name.toLowerCase());
|
||||
|
||||
/** Match an ASCII header name without allocating a potentially enormous line. */
|
||||
export const isCaptureCredentialHeaderBytes = (
|
||||
bytes: Uint8Array,
|
||||
start: number,
|
||||
end: number,
|
||||
): boolean => {
|
||||
for (const name of CREDENTIAL_HEADERS) {
|
||||
for (const name of TRANSPORT_CREDENTIAL_HEADERS) {
|
||||
if (end - start !== name.length) continue;
|
||||
let matches = true;
|
||||
for (let index = 0; index < name.length; index++) {
|
||||
@@ -52,12 +48,7 @@ export class CaptureRedaction {
|
||||
: undefined;
|
||||
this.#requestUrl =
|
||||
typeof request?.url === "string" ? request.url : undefined;
|
||||
// An unpaired surrogate has no UTF-8 representation. Buffer.from would
|
||||
// invent replacement-character bytes and incorrectly discard real evidence.
|
||||
this.#sensitiveByteValues = sensitiveValues.flatMap((value) => {
|
||||
const encoded = Buffer.from(value, "utf8");
|
||||
return encoded.toString("utf8") === value ? [encoded] : [];
|
||||
});
|
||||
this.#sensitiveByteValues = explicitUtf8Spellings(sensitiveValues);
|
||||
}
|
||||
|
||||
/** Preserve exact text unless explicitly sensitive or known transport URL userinfo. */
|
||||
@@ -76,9 +67,8 @@ export class CaptureRedaction {
|
||||
}
|
||||
}
|
||||
if (
|
||||
this.sensitiveValues.some(
|
||||
(literal) => value.includes(literal) || result.includes(literal),
|
||||
)
|
||||
explicitValuesOccurIn(value, this.sensitiveValues) ||
|
||||
explicitValuesOccurIn(result, this.sensitiveValues)
|
||||
) {
|
||||
this.redactions.push({ pointer, reason: "explicit-sensitive-value" });
|
||||
return null;
|
||||
@@ -116,7 +106,7 @@ export class CaptureRedaction {
|
||||
return (
|
||||
(/^\/(request|response)\/headers\/\d+\/value$/.test(pointer) &&
|
||||
typeof parent.name === "string" &&
|
||||
isCaptureCredentialHeader(parent.name)) ||
|
||||
isTransportCredentialHeader(parent.name)) ||
|
||||
/^\/(request|response)\/cookies\/\d+\/value$/.test(pointer)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -12,8 +12,8 @@ import {
|
||||
CaptureRedaction,
|
||||
captureObject,
|
||||
capturePointer,
|
||||
isCaptureCredentialHeader,
|
||||
} from "./CaptureRedaction.js";
|
||||
import { isTransportCredentialHeader } from "../../domain/explicitSensitiveValues.js";
|
||||
import { QLOG_CAPTURE_PROVIDER_IDENTITY } from "./CaptureProviderIdentities.js";
|
||||
|
||||
const MAX_VARINT = (1n << 62n) - 1n;
|
||||
@@ -430,7 +430,7 @@ const project = (
|
||||
/^\/data\/frame\/headers\/\d+\/value$/u.test(pointer) &&
|
||||
typeof parent?.name === "string";
|
||||
if (
|
||||
(header && isCaptureCredentialHeader(String(parent?.name))) ||
|
||||
(header && isTransportCredentialHeader(String(parent?.name))) ||
|
||||
qlogToken(eventName, pointer, parent)
|
||||
)
|
||||
return redactor.credential(pointer);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
|
||||
import { isTransportCredentialHeader } from "../../domain/explicitSensitiveValues.js";
|
||||
import {
|
||||
isJson,
|
||||
mediaTypeEssence,
|
||||
@@ -54,13 +55,6 @@ export type NetworkBodyPreflight = (input: {
|
||||
readonly mediaType: string | undefined;
|
||||
}) => BrowserScenarioEventAdmission | Promise<BrowserScenarioEventAdmission>;
|
||||
|
||||
const credentialHeaders = new Set([
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"cookie",
|
||||
"set-cookie",
|
||||
]);
|
||||
|
||||
const errorMessage = (
|
||||
cause: unknown,
|
||||
secrets: BrowserScenarioSecrets,
|
||||
@@ -95,7 +89,7 @@ export const captureNetworkHeaders = async (
|
||||
{ state: "captured" }
|
||||
>["items"] = [];
|
||||
for (const { name, value } of headers) {
|
||||
const credential = credentialHeaders.has(name.toLowerCase());
|
||||
const credential = isTransportCredentialHeader(name);
|
||||
cost +=
|
||||
secrets.redactionAllocationCost(
|
||||
name.length + (credential ? 0 : value.length),
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
} from "./ComputerHomeFiles.js";
|
||||
import { DRAWN_WAIT_MS, drawnScreenshot } from "./DrawnScreenshot.js";
|
||||
import { awaitHttpReply, type HttpReply } from "./HttpReply.js";
|
||||
import { redactExplicitText } from "../domain/explicitSensitiveValues.js";
|
||||
|
||||
const READY_TIMEOUT_MS = 180_000;
|
||||
const REQUEST_TIMEOUT_MS = 120_000;
|
||||
@@ -697,7 +698,9 @@ export class AgentEnvComputer {
|
||||
redacted(text: string): string {
|
||||
return this.#token === null
|
||||
? text
|
||||
: text.replaceAll(this.#token, "[control token]");
|
||||
: redactExplicitText(text, [this.#token], {
|
||||
replacement: "[control token]",
|
||||
});
|
||||
}
|
||||
|
||||
/** A framework-only route of the computer, with its control token; its answer is read within `bounds.limit`. */
|
||||
|
||||
@@ -3,6 +3,11 @@ import { z } from "zod";
|
||||
import type { JsonValue } from "../jsonValue.js";
|
||||
import { gunzip, isGzip } from "./gzipDocument.js";
|
||||
import { decodeStateText, jsonStringChunks } from "./stateFormats.js";
|
||||
import {
|
||||
explicitTextSpellings,
|
||||
explicitUtf16Spellings,
|
||||
lowercasePercentEscapes,
|
||||
} from "../explicitSensitiveValues.js";
|
||||
|
||||
/** A secret's handle: the name of one file in the build host's secrets folder. */
|
||||
export const buildSecretHandleSchema = z
|
||||
@@ -50,30 +55,13 @@ export const secretValueRefusal = (
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/** The characters of the base64 encoding of `bytes` that depend on them alone, when they start `offset` bytes into a longer encoding. */
|
||||
const base64Stable = (bytes: Buffer, offset: number): string => {
|
||||
const encoded = Buffer.concat([Buffer.alloc(offset), bytes]).toString(
|
||||
"base64",
|
||||
);
|
||||
return encoded.slice(
|
||||
Math.ceil((8 * offset) / 6),
|
||||
Math.floor((8 * (offset + bytes.length)) / 6),
|
||||
);
|
||||
};
|
||||
|
||||
const jsonEscaped = (text: string) => JSON.stringify(text).slice(1, -1);
|
||||
const jsonEscaped = (text: string): string =>
|
||||
explicitTextSpellings(text, ["json-escaped"])[0] ?? text;
|
||||
const xmlEscaped = (text: string) =>
|
||||
text.replaceAll("&", "&").replaceAll("<", "<");
|
||||
|
||||
/** The value's spellings as text: as it is, escaped in XML and in JSON once or twice, in hex, in base64 at each alignment and in URL and form encoding. */
|
||||
/** The value's spellings as text: as it is, escaped in XML and in JSON once or twice, in hex, in base64 at each alignment and in URL and form encoding with either escape case. */
|
||||
const textSpellings = (value: string): readonly string[] => {
|
||||
const bytes = Buffer.from(value, "utf8");
|
||||
const hex = bytes.toString("hex");
|
||||
const base64 = [0, 1, 2].map((offset) => base64Stable(bytes, offset));
|
||||
const url = encodeURIComponent(value);
|
||||
const form = new URLSearchParams([["value", value]])
|
||||
.toString()
|
||||
.slice("value=".length);
|
||||
const escapedXml = xmlEscaped(value);
|
||||
const xml = [
|
||||
value,
|
||||
@@ -82,33 +70,28 @@ const textSpellings = (value: string): readonly string[] => {
|
||||
text.replaceAll('"', """),
|
||||
]),
|
||||
];
|
||||
const percent = explicitTextSpellings(value, [
|
||||
"uri-component",
|
||||
"uri-plus-space",
|
||||
"form",
|
||||
]);
|
||||
return [
|
||||
...new Set([
|
||||
...xml,
|
||||
...xml.map(jsonEscaped),
|
||||
...xml.map((text) => jsonEscaped(jsonEscaped(text))),
|
||||
hex,
|
||||
hex.toUpperCase(),
|
||||
...base64,
|
||||
...base64.map((text) => text.replaceAll("+", "-").replaceAll("/", "_")),
|
||||
url,
|
||||
url.replaceAll("%20", "+"),
|
||||
url.replace(/%[0-9A-F]{2}/gu, (escape) => escape.toLowerCase()),
|
||||
form,
|
||||
form.replace(/%[0-9A-F]{2}/gu, (escape) => escape.toLowerCase()),
|
||||
...explicitTextSpellings(value, ["hex", "base64"]),
|
||||
...percent,
|
||||
...percent.map(lowercasePercentEscapes),
|
||||
]),
|
||||
];
|
||||
};
|
||||
|
||||
/** The byte spellings a scan looks for: each text spelling in UTF-8, and the value in UTF-16 of either byte order. */
|
||||
const byteSpellings = (value: string): readonly Buffer[] => {
|
||||
const utf16 = Buffer.from(value, "utf16le");
|
||||
return [
|
||||
...textSpellings(value).map((text) => Buffer.from(text, "utf8")),
|
||||
utf16,
|
||||
Buffer.from(utf16).swap16(),
|
||||
];
|
||||
};
|
||||
const byteSpellings = (value: string): readonly Buffer[] => [
|
||||
...textSpellings(value).map((text) => Buffer.from(text, "utf8")),
|
||||
...explicitUtf16Spellings(value),
|
||||
];
|
||||
|
||||
const holds = (bytes: Uint8Array, spellings: readonly Buffer[]): boolean => {
|
||||
const haystack = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length);
|
||||
|
||||
@@ -1,15 +1,22 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { redactExplicitText } from "./explicitSensitiveValues.js";
|
||||
import {
|
||||
explicitTextSpellings,
|
||||
explicitUtf16Spellings,
|
||||
explicitUtf8Spellings,
|
||||
explicitValueSpans,
|
||||
lowercasePercentEscapes,
|
||||
explicitValuesOccurIn,
|
||||
isTransportCredentialHeader,
|
||||
redactExplicitOutput,
|
||||
redactExplicitText,
|
||||
} from "./explicitSensitiveValues.js";
|
||||
|
||||
it.each([
|
||||
["secret context", ["secret"], "[REDACTED] context"],
|
||||
["secret context", ["secret", "REDACTED"], "… context"],
|
||||
["secret context", ["secret", "[", "…"], " context"],
|
||||
[
|
||||
"secretable context",
|
||||
["secret", "able", "table"],
|
||||
"[REDACTED][REDACTED] context",
|
||||
],
|
||||
// Overlapping declarations redact as one span, leaving no fragment between.
|
||||
["secretable context", ["secret", "able", "table"], "[REDACTED] context"],
|
||||
["secretsecret context", ["secret", "…", "REDACTED"], " context"],
|
||||
] as const)(
|
||||
"keeps declared literals out of replacement text: %s",
|
||||
@@ -24,4 +31,144 @@ it("does not guess sensitivity or treat empty declarations as matches", () => {
|
||||
expect(redactExplicitText("API_SECRET ordinary evidence", [""])).toBe(
|
||||
"API_SECRET ordinary evidence",
|
||||
);
|
||||
expect(explicitValuesOccurIn("API_SECRET ordinary evidence", [""])).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("uses a caller's marker unless it would spell a declaration", () => {
|
||||
expect(
|
||||
redactExplicitText("token=abc", ["abc"], {
|
||||
replacement: "[control token]",
|
||||
}),
|
||||
).toBe("token=[control token]");
|
||||
expect(
|
||||
redactExplicitText("token=con", ["con"], {
|
||||
replacement: "[control token]",
|
||||
}),
|
||||
).toBe("token=[REDACTED]");
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"complete output keeps an unrelated tail",
|
||||
false,
|
||||
"log private",
|
||||
"log private",
|
||||
],
|
||||
[
|
||||
"cut output replaces a declared prefix",
|
||||
true,
|
||||
"log private-tr",
|
||||
"log [REDACTED]",
|
||||
],
|
||||
[
|
||||
"cut output replaces whole values before the tail",
|
||||
true,
|
||||
"private-transport-token then private-",
|
||||
"[REDACTED] then [REDACTED]",
|
||||
],
|
||||
[
|
||||
"cut output without a declared prefix is unchanged",
|
||||
true,
|
||||
"log done",
|
||||
"log done",
|
||||
],
|
||||
] as const)("%s", (_name, cutOff, text, expected) => {
|
||||
expect(
|
||||
redactExplicitText(text, ["private-transport-token"], { cutOff }),
|
||||
).toBe(expected);
|
||||
});
|
||||
|
||||
it("treats output as cut when more bytes were observed than retained", () => {
|
||||
const token = "private-transport-token";
|
||||
expect(
|
||||
redactExplicitOutput(
|
||||
{ text: token.slice(0, 8), bytes: 8, observedBytes: token.length },
|
||||
[token],
|
||||
),
|
||||
).toBe("[REDACTED]");
|
||||
expect(
|
||||
redactExplicitOutput({ text: "priv", bytes: 4, observedBytes: 4 }, [token]),
|
||||
).toBe("priv");
|
||||
});
|
||||
|
||||
it("spells declarations in UTF-8 and skips values with no UTF-8 form", () => {
|
||||
expect(explicitUtf8Spellings(["é", "\ud800", "", "é"])).toEqual([
|
||||
Buffer.from("é", "utf8"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("recognizes protocol-defined credential headers in any case only", () => {
|
||||
for (const name of [
|
||||
"Authorization",
|
||||
"proxy-authorization",
|
||||
"Cookie",
|
||||
"SET-COOKIE",
|
||||
])
|
||||
expect(isTransportCredentialHeader(name)).toBe(true);
|
||||
for (const name of ["x-api-key", "x-auth-token", "location"])
|
||||
expect(isTransportCredentialHeader(name)).toBe(false);
|
||||
});
|
||||
|
||||
it("merges overlapping occurrences of one or several declarations", () => {
|
||||
expect([...explicitValueSpans("aaaa", ["aa"])]).toEqual([
|
||||
{ start: 0, end: 4 },
|
||||
]);
|
||||
expect([
|
||||
...explicitValueSpans("abcdef xx cd", ["abcd", "cdef", "cd"]),
|
||||
]).toEqual([
|
||||
{ start: 0, end: 6 },
|
||||
{ start: 10, end: 12 },
|
||||
]);
|
||||
expect([...explicitValueSpans("abab", ["ab"])]).toEqual([
|
||||
{ start: 0, end: 2 },
|
||||
{ start: 2, end: 4 },
|
||||
]);
|
||||
expect([...explicitValueSpans("text", [""])]).toEqual([]);
|
||||
});
|
||||
|
||||
it("spells a declared value in each selected encoding", () => {
|
||||
expect(
|
||||
explicitTextSpellings("a b'/é", [
|
||||
"uri-component",
|
||||
"uri-query",
|
||||
"uri-plus-space",
|
||||
"form",
|
||||
]),
|
||||
).toEqual([
|
||||
"a%20b'%2F%C3%A9",
|
||||
"a%20b%27%2F%C3%A9",
|
||||
"a+b'%2F%C3%A9",
|
||||
"a+b%27%2F%C3%A9",
|
||||
]);
|
||||
expect(explicitTextSpellings('q"\\n', ["json-escaped"])).toEqual([
|
||||
'q\\"\\\\n',
|
||||
]);
|
||||
expect(explicitTextSpellings("ok", ["json-escaped"])).toEqual([]);
|
||||
expect(explicitTextSpellings("\u00ff", ["hex"])).toEqual(["c3bf", "C3BF"]);
|
||||
const base64 = explicitTextSpellings("secret-value", ["base64"]);
|
||||
for (const prefix of ["", "x", "xy"])
|
||||
expect(
|
||||
base64.some((spelling) =>
|
||||
Buffer.from(`${prefix}secret-value`)
|
||||
.toString("base64")
|
||||
.includes(spelling),
|
||||
),
|
||||
).toBe(true);
|
||||
// URL serializers write an unpaired surrogate as U+FFFD.
|
||||
expect(explicitTextSpellings("\ud800", ["uri-component"])).toEqual([
|
||||
"%EF%BF%BD",
|
||||
]);
|
||||
expect(explicitUtf16Spellings("ab")).toEqual([
|
||||
Buffer.from([0x61, 0, 0x62, 0]),
|
||||
Buffer.from([0, 0x61, 0, 0x62]),
|
||||
]);
|
||||
expect(lowercasePercentEscapes("a%2Fb%C3")).toBe("a%2fb%c3");
|
||||
});
|
||||
|
||||
it("finds spans lazily, so a budgeted caller stops matching early", () => {
|
||||
const spans = explicitValueSpans("token ".repeat(100_000), ["token"]);
|
||||
expect(spans.next().value).toEqual({ start: 0, end: 5 });
|
||||
expect(spans.next().value).toEqual({ start: 6, end: 11 });
|
||||
});
|
||||
@@ -1,20 +1,260 @@
|
||||
/** Replace caller-declared literal text values without guessing sensitivity from names. */
|
||||
import { toUSVString } from "node:util";
|
||||
|
||||
/**
|
||||
* The one owner for redacting declared sensitive values: literals a caller
|
||||
* explicitly marked sensitive, and transport credentials REA minted itself.
|
||||
* Nothing here infers secrecy from names or text patterns; an undeclared value
|
||||
* is ordinary evidence. Producers choose what they declare, this module
|
||||
* decides how every declared value is found and replaced.
|
||||
*/
|
||||
|
||||
/**
|
||||
* HTTP fields whose values are transport credentials by protocol definition
|
||||
* (RFC 9110 authentication, RFC 6265 cookies), in lower case. Other headers
|
||||
* are ordinary evidence unless a caller declares them.
|
||||
*/
|
||||
export const TRANSPORT_CREDENTIAL_HEADERS: ReadonlySet<string> = new Set([
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"cookie",
|
||||
"set-cookie",
|
||||
]);
|
||||
|
||||
/** Whether an HTTP field name carries a transport credential. */
|
||||
export const isTransportCredentialHeader = (name: string): boolean =>
|
||||
TRANSPORT_CREDENTIAL_HEADERS.has(name.toLowerCase());
|
||||
|
||||
/** Distinct non-empty declarations, longest first so overlaps redact whole. */
|
||||
const declaredLiterals = (values: readonly string[]): readonly string[] =>
|
||||
[...new Set(values)]
|
||||
.filter((literal) => literal !== "")
|
||||
.sort((left, right) => right.length - left.length);
|
||||
|
||||
/** Whether any declared literal occurs in `value`. */
|
||||
export const explicitValuesOccurIn = (
|
||||
value: string,
|
||||
sensitiveValues: readonly string[],
|
||||
): boolean =>
|
||||
sensitiveValues.some((literal) => literal !== "" && value.includes(literal));
|
||||
|
||||
/**
|
||||
* UTF-8 spellings of the declared literals, for searching retained bytes. An
|
||||
* unpaired surrogate has no UTF-8 representation; encoding it would invent
|
||||
* replacement-character bytes and discard real evidence, so it is skipped.
|
||||
*/
|
||||
export const explicitUtf8Spellings = (
|
||||
sensitiveValues: readonly string[],
|
||||
): readonly Buffer[] =>
|
||||
declaredLiterals(sensitiveValues).flatMap((value) => {
|
||||
const encoded = Buffer.from(value, "utf8");
|
||||
return encoded.toString("utf8") === value ? [encoded] : [];
|
||||
});
|
||||
|
||||
/** One occurrence of declared text, as a half-open UTF-16 range. */
|
||||
export type ExplicitValueSpan = Readonly<{ start: number; end: number }>;
|
||||
|
||||
/**
|
||||
* Every occurrence of any declared literal in `value`, in order. Overlapping
|
||||
* occurrences (of one literal or of several) merge into one span, so no part
|
||||
* of an overlapping declaration survives between two replacements. Spans are
|
||||
* produced lazily with one cursor per literal, so a caller that stops at a
|
||||
* budget does no further matching work.
|
||||
*/
|
||||
export function* explicitValueSpans(
|
||||
value: string,
|
||||
sensitiveValues: readonly string[],
|
||||
): Generator<ExplicitValueSpan, void, undefined> {
|
||||
const literals = declaredLiterals(sensitiveValues);
|
||||
const next = literals.map((literal) => value.indexOf(literal));
|
||||
let pending: ExplicitValueSpan | undefined;
|
||||
for (;;) {
|
||||
// Earliest occurrence; at one start, the longest literal (listed first).
|
||||
let selected = -1;
|
||||
for (let index = 0; index < literals.length; index += 1)
|
||||
if (next[index]! >= 0 && (selected < 0 || next[index]! < next[selected]!))
|
||||
selected = index;
|
||||
if (selected < 0) break;
|
||||
const start = next[selected]!;
|
||||
const end = start + literals[selected]!.length;
|
||||
next[selected] = value.indexOf(literals[selected]!, start + 1);
|
||||
if (pending !== undefined && start < pending.end)
|
||||
pending = { start: pending.start, end: Math.max(pending.end, end) };
|
||||
else {
|
||||
if (pending !== undefined) yield pending;
|
||||
pending = { start, end };
|
||||
}
|
||||
}
|
||||
if (pending !== undefined) yield pending;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encoded spellings a declared value can take in retained evidence:
|
||||
* - `uri-component`: `encodeURIComponent`;
|
||||
* - `uri-query`: a WHATWG special-URL query, which also escapes `'`;
|
||||
* - `uri-plus-space`: a URI component whose spaces are written `+`;
|
||||
* - `form`: `application/x-www-form-urlencoded`;
|
||||
* - `json-escaped`: the body of a JSON string literal;
|
||||
* - `hex`: UTF-8 bytes in lower- and upper-case hex;
|
||||
* - `base64`: UTF-8 bytes in base64 and base64url at each of the three byte
|
||||
* alignments inside a longer encoding (only the characters that depend on
|
||||
* the value alone).
|
||||
* Percent escapes are upper case; see {@link lowercasePercentEscapes}.
|
||||
*/
|
||||
export type ExplicitSpelling =
|
||||
| "uri-component"
|
||||
| "uri-query"
|
||||
| "uri-plus-space"
|
||||
| "form"
|
||||
| "json-escaped"
|
||||
| "hex"
|
||||
| "base64";
|
||||
|
||||
/** Characters of `bytes`' base64 that depend on them alone at a byte offset. */
|
||||
const base64Stable = (bytes: Buffer, offset: number): string => {
|
||||
const encoded = Buffer.concat([Buffer.alloc(offset), bytes]).toString(
|
||||
"base64",
|
||||
);
|
||||
return encoded.slice(
|
||||
Math.ceil((8 * offset) / 6),
|
||||
Math.floor((8 * (offset + bytes.length)) / 6),
|
||||
);
|
||||
};
|
||||
|
||||
const spellingsOf = (value: string, spelling: ExplicitSpelling): string[] => {
|
||||
// URL and form serializers replace unpaired surrogates with U+FFFD.
|
||||
const scalar = toUSVString(value);
|
||||
switch (spelling) {
|
||||
case "uri-component":
|
||||
return [encodeURIComponent(scalar)];
|
||||
case "uri-query":
|
||||
return [encodeURIComponent(scalar).replace(/'/gu, "%27")];
|
||||
case "uri-plus-space":
|
||||
return [encodeURIComponent(scalar).replaceAll("%20", "+")];
|
||||
case "form":
|
||||
return [
|
||||
new URLSearchParams([["value", scalar]])
|
||||
.toString()
|
||||
.slice("value=".length),
|
||||
];
|
||||
case "json-escaped":
|
||||
return [JSON.stringify(value).slice(1, -1)];
|
||||
case "hex": {
|
||||
const hex = Buffer.from(value, "utf8").toString("hex");
|
||||
return [hex, hex.toUpperCase()];
|
||||
}
|
||||
case "base64": {
|
||||
const bytes = Buffer.from(value, "utf8");
|
||||
const aligned = [0, 1, 2].map((offset) => base64Stable(bytes, offset));
|
||||
return [
|
||||
...aligned,
|
||||
...aligned.map((text) =>
|
||||
text.replaceAll("+", "-").replaceAll("/", "_"),
|
||||
),
|
||||
];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* The selected encoded spellings of one declared value, distinct and
|
||||
* non-empty, excluding the value itself.
|
||||
*/
|
||||
export const explicitTextSpellings = (
|
||||
value: string,
|
||||
spellings: readonly ExplicitSpelling[],
|
||||
): readonly string[] => [
|
||||
...new Set(
|
||||
spellings
|
||||
.flatMap((spelling) => spellingsOf(value, spelling))
|
||||
.filter((text) => text !== "" && text !== value),
|
||||
),
|
||||
];
|
||||
|
||||
/** The value in UTF-16 of both byte orders, for searching retained bytes. */
|
||||
export const explicitUtf16Spellings = (value: string): readonly Buffer[] => {
|
||||
const little = Buffer.from(value, "utf16le");
|
||||
return [little, Buffer.from(little).swap16()];
|
||||
};
|
||||
|
||||
/** Lower-case every percent escape, so spellings compare escape-case-insensitively. */
|
||||
export const lowercasePercentEscapes = (value: string): string =>
|
||||
value.includes("%")
|
||||
? value.replace(/%[0-9a-f]{2}/giu, (escape) => escape.toLowerCase())
|
||||
: value;
|
||||
|
||||
export type ExplicitRedactionOptions = Readonly<{
|
||||
/** Preferred marker; a safe fallback is used when it contains a declaration. */
|
||||
replacement?: string;
|
||||
/**
|
||||
* The text was cut by an output budget, so its end may hold the start of a
|
||||
* declared value. The longest such prefix is replaced as well.
|
||||
*/
|
||||
cutOff?: boolean;
|
||||
}>;
|
||||
|
||||
/** Replace declared literal text values without guessing sensitivity from names. */
|
||||
export const redactExplicitText = (
|
||||
value: string,
|
||||
sensitiveValues: readonly string[],
|
||||
options: ExplicitRedactionOptions = {},
|
||||
): string => {
|
||||
const literals = [...new Set(sensitiveValues)].filter(
|
||||
(literal) => literal !== "",
|
||||
);
|
||||
const literals = declaredLiterals(sensitiveValues);
|
||||
if (literals.length === 0) return value;
|
||||
const replacement =
|
||||
["[REDACTED]", "…", ""].find((candidate) =>
|
||||
[
|
||||
...(options.replacement === undefined ? [] : [options.replacement]),
|
||||
"[REDACTED]",
|
||||
"…",
|
||||
"",
|
||||
].find((candidate) =>
|
||||
literals.every((literal) => !candidate.includes(literal)),
|
||||
) ?? "";
|
||||
let result = value;
|
||||
for (const literal of literals.sort(
|
||||
(left, right) => right.length - left.length,
|
||||
))
|
||||
result = result.replaceAll(literal, replacement);
|
||||
let result = "";
|
||||
let cursor = 0;
|
||||
for (const { start, end } of explicitValueSpans(value, literals)) {
|
||||
result += value.slice(cursor, start) + replacement;
|
||||
cursor = end;
|
||||
}
|
||||
result += value.slice(cursor);
|
||||
if (options.cutOff === true) {
|
||||
const tail = cutOffPrefixLength(result, literals);
|
||||
if (tail > 0)
|
||||
result = `${result.slice(0, result.length - tail)}${replacement}`;
|
||||
}
|
||||
// Replacement boundaries can form another declaration; exclude that text.
|
||||
return literals.some((literal) => result.includes(literal)) ? "" : result;
|
||||
};
|
||||
|
||||
/**
|
||||
* Redact declared values in retained output. Output whose observed length
|
||||
* exceeds the retained bytes was cut by its budget, so its end is checked for
|
||||
* the start of a declared value too.
|
||||
*/
|
||||
export const redactExplicitOutput = (
|
||||
output: Readonly<{ text: string; bytes: number; observedBytes: number }>,
|
||||
sensitiveValues: readonly string[],
|
||||
replacement?: string,
|
||||
): string =>
|
||||
redactExplicitText(output.text, sensitiveValues, {
|
||||
cutOff: output.observedBytes > output.bytes,
|
||||
...(replacement === undefined ? {} : { replacement }),
|
||||
});
|
||||
|
||||
/** Length of the longest proper prefix of a declaration that ends `text`. */
|
||||
const cutOffPrefixLength = (
|
||||
text: string,
|
||||
literals: readonly string[],
|
||||
): number => {
|
||||
let longest = 0;
|
||||
for (const literal of literals)
|
||||
for (
|
||||
let length = Math.min(text.length, literal.length - 1);
|
||||
length > longest;
|
||||
length -= 1
|
||||
)
|
||||
if (text.endsWith(literal.slice(0, length))) {
|
||||
longest = length;
|
||||
break;
|
||||
}
|
||||
return longest;
|
||||
};
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { isValidFridaRemoteAddress } from "../domain/fridaRemoteAddress.js";
|
||||
import { redactExplicitFailure } from "../domain/explicitSensitiveFailure.js";
|
||||
import { explicitValueSpans } from "../domain/explicitSensitiveValues.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
@@ -1392,59 +1393,45 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort {
|
||||
}
|
||||
}
|
||||
|
||||
/** Redact declared values, then keep what fits `maximumBytes` of JSON string body. */
|
||||
const boundedRedactedString = (
|
||||
input: string,
|
||||
maximumBytes: number,
|
||||
secrets: readonly string[],
|
||||
): { readonly value: string; readonly truncated: boolean } => {
|
||||
const replacement = "[redacted]";
|
||||
const replacementBytes = jsonStringByteLength(replacement) - 2;
|
||||
const chunks: string[] = [];
|
||||
let outputBytes = 0;
|
||||
let cursor = 0;
|
||||
let segmentStart = 0;
|
||||
let truncated = false;
|
||||
while (cursor < input.length) {
|
||||
let nextSecretStart = input.length;
|
||||
let nextSecret: string | undefined;
|
||||
for (const secret of secrets) {
|
||||
const match = input.indexOf(secret, cursor);
|
||||
if (match !== -1 && match < nextSecretStart) {
|
||||
nextSecretStart = match;
|
||||
nextSecret = secret;
|
||||
}
|
||||
}
|
||||
if (nextSecretStart === cursor && nextSecret !== undefined) {
|
||||
const replacement = "[redacted]";
|
||||
const replacementBytes = jsonStringByteLength(replacement) - 2;
|
||||
if (outputBytes + replacementBytes > maximumBytes) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
chunks.push(input.slice(segmentStart, cursor), replacement);
|
||||
outputBytes += replacementBytes;
|
||||
cursor += nextSecret.length;
|
||||
segmentStart = cursor;
|
||||
continue;
|
||||
}
|
||||
while (cursor < nextSecretStart) {
|
||||
const keep = (end: number): boolean => {
|
||||
const start = cursor;
|
||||
while (cursor < end) {
|
||||
const codePoint = input.codePointAt(cursor);
|
||||
if (codePoint === undefined) break;
|
||||
const character = String.fromCodePoint(codePoint);
|
||||
const characterBytes = jsonStringByteLength(character) - 2;
|
||||
if (
|
||||
cursor + character.length > nextSecretStart ||
|
||||
cursor + character.length > end ||
|
||||
outputBytes + characterBytes > maximumBytes
|
||||
) {
|
||||
truncated = true;
|
||||
)
|
||||
break;
|
||||
}
|
||||
outputBytes += characterBytes;
|
||||
cursor += character.length;
|
||||
}
|
||||
if (truncated) break;
|
||||
chunks.push(input.slice(start, cursor));
|
||||
return cursor === end;
|
||||
};
|
||||
for (const span of explicitValueSpans(input, secrets)) {
|
||||
if (!keep(span.start)) return { value: chunks.join(""), truncated: true };
|
||||
if (outputBytes + replacementBytes > maximumBytes)
|
||||
return { value: chunks.join(""), truncated: true };
|
||||
chunks.push(replacement);
|
||||
outputBytes += replacementBytes;
|
||||
cursor = span.end;
|
||||
}
|
||||
chunks.push(input.slice(segmentStart, cursor));
|
||||
if (cursor < input.length) truncated = true;
|
||||
return { value: chunks.join(""), truncated };
|
||||
const complete = keep(input.length);
|
||||
return { value: chunks.join(""), truncated: !complete };
|
||||
};
|
||||
|
||||
type BoundedJson = {
|
||||
|
||||
@@ -104,7 +104,7 @@ import {
|
||||
RegularFileChangedError,
|
||||
sameRegularFileState,
|
||||
} from "../filesystem/RegularFile.js";
|
||||
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
|
||||
import { redactExplicitText } from "../domain/explicitSensitiveValues.js";
|
||||
|
||||
export type GhidraTargetSnapshot =
|
||||
| { readonly kind: "captured"; readonly bytes: Buffer }
|
||||
@@ -1537,15 +1537,9 @@ export class GhidraClient {
|
||||
}
|
||||
|
||||
#redactAuthentication(value: string, incomplete = false): string {
|
||||
for (const token of this.#authenticationTokens) {
|
||||
const bytes = Buffer.byteLength(value);
|
||||
value = redactCapturedTransportCredential(
|
||||
{ text: value, bytes, observedBytes: bytes + (incomplete ? 1 : 0) },
|
||||
token,
|
||||
"[REDACTED]",
|
||||
);
|
||||
}
|
||||
return value;
|
||||
return redactExplicitText(value, [...this.#authenticationTokens], {
|
||||
cutOff: incomplete,
|
||||
});
|
||||
}
|
||||
|
||||
#onProcessDiagnostic(event: ProviderProcessDiagnostic): void {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { utf8TailExcerpt } from "../domain/utf8Excerpt.js";
|
||||
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
|
||||
import {
|
||||
redactExplicitOutput,
|
||||
redactExplicitText,
|
||||
} from "../domain/explicitSensitiveValues.js";
|
||||
import type {
|
||||
ProviderProcessOutputSince,
|
||||
ProviderProcessSnapshot,
|
||||
@@ -141,23 +144,19 @@ export const ghidraFailureOutput = (
|
||||
complete: scoped.complete && !drainIncomplete,
|
||||
launcher_failures: observedLauncherFailures(scoped, token),
|
||||
observed_oome: (scoped.markers ?? []).map((marker) => {
|
||||
const bytes = Buffer.byteLength(marker.line, "utf8");
|
||||
// The supervisor's runtime owns which memory area the line names.
|
||||
const { area } = runtimeExhaustion("jvm", marker.stream, marker);
|
||||
return {
|
||||
stream: marker.stream,
|
||||
offset: marker.offset,
|
||||
...(area === undefined ? {} : { area }),
|
||||
line: redactCapturedTransportCredential(
|
||||
// A cut line may end in a partial credential.
|
||||
line: redactExplicitText(
|
||||
marker.line,
|
||||
token === undefined ? [] : [token],
|
||||
{
|
||||
text: marker.line,
|
||||
bytes,
|
||||
// A cut line may end in a partial credential.
|
||||
observedBytes:
|
||||
marker.trailingContextIncomplete === true ? bytes + 1 : bytes,
|
||||
cutOff: marker.trailingContextIncomplete === true,
|
||||
},
|
||||
token,
|
||||
"[REDACTED]",
|
||||
),
|
||||
line_complete: marker.trailingContextIncomplete !== true,
|
||||
...(marker.truncated ? { truncated: true } : {}),
|
||||
@@ -240,10 +239,9 @@ const observedLauncherFailures = (
|
||||
const failures: GhidraLauncherFailure[] = [];
|
||||
let truncated = false;
|
||||
for (const stream of ["stdout", "stderr"] as const) {
|
||||
const text = redactCapturedTransportCredential(
|
||||
const text = redactExplicitOutput(
|
||||
scoped.snapshot[stream],
|
||||
token,
|
||||
"[REDACTED]",
|
||||
token === undefined ? [] : [token],
|
||||
);
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
const failure = ghidraLauncherFailure(stream, line);
|
||||
@@ -267,7 +265,7 @@ export const ghidraDiagnosticStream = (
|
||||
): JsonValue => {
|
||||
// Redact the whole retained text first, so the cut cannot expose part of a token.
|
||||
const excerpt = utf8TailExcerpt(
|
||||
redactCapturedTransportCredential(stream, token, "[REDACTED]"),
|
||||
redactExplicitOutput(stream, token === undefined ? [] : [token]),
|
||||
GHIDRA_DIAGNOSTIC_EXCERPT_BYTES,
|
||||
);
|
||||
return {
|
||||
|
||||
@@ -48,7 +48,7 @@ import {
|
||||
execFileFailureCapturedOutput,
|
||||
} from "../process/ExecFileOutput.js";
|
||||
import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js";
|
||||
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
|
||||
import { redactExplicitText } from "../domain/explicitSensitiveValues.js";
|
||||
import {
|
||||
cleanupOwnedProcessGroup,
|
||||
type ProcessCleanupResult,
|
||||
@@ -517,12 +517,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher {
|
||||
environment,
|
||||
onStderr: (mode, stderr) => {
|
||||
if (stderr === "") return;
|
||||
const bytes = Buffer.byteLength(stderr);
|
||||
const redacted = redactCapturedTransportCredential(
|
||||
{ text: stderr, bytes, observedBytes: bytes },
|
||||
session.token,
|
||||
"[REDACTED]",
|
||||
);
|
||||
const redacted = redactExplicitText(stderr, [session.token]);
|
||||
const bounded = utf8HeadExcerpt(
|
||||
redacted,
|
||||
GHIDRA_LAUNCH_COMMAND_OUTPUT_BYTES,
|
||||
@@ -530,7 +525,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher {
|
||||
launchSupport.push({
|
||||
mode,
|
||||
stderr: bounded.text,
|
||||
stderr_bytes: bytes,
|
||||
stderr_bytes: Buffer.byteLength(stderr),
|
||||
stderr_truncated: bounded.truncated,
|
||||
});
|
||||
},
|
||||
|
||||
@@ -61,7 +61,7 @@ import {
|
||||
ProviderStartupDeadline,
|
||||
waitForAbortableDelay,
|
||||
} from "../process/ProviderDeadline.js";
|
||||
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
|
||||
import { redactExplicitOutput } from "../domain/explicitSensitiveValues.js";
|
||||
import { ProviderRunLineage } from "../process/ProviderRunLineage.js";
|
||||
import { AbortedWait, waitForAbortable } from "../process/AbortablePromise.js";
|
||||
import { SharedStartupAttempt } from "../process/SharedStartupAttempt.js";
|
||||
@@ -1116,18 +1116,18 @@ export class HopperClient<
|
||||
exit_code: snapshot.exitCode ?? null,
|
||||
signal: snapshot.signal ?? null,
|
||||
stdout: {
|
||||
text: redactCapturedTransportCredential(
|
||||
text: redactExplicitOutput(
|
||||
snapshot.stdout,
|
||||
token,
|
||||
token === undefined ? [] : [token],
|
||||
"[redacted transport credential]",
|
||||
),
|
||||
bytes: snapshot.stdout.observedBytes,
|
||||
retained_bytes: snapshot.stdout.bytes,
|
||||
},
|
||||
stderr: {
|
||||
text: redactCapturedTransportCredential(
|
||||
text: redactExplicitOutput(
|
||||
snapshot.stderr,
|
||||
token,
|
||||
token === undefined ? [] : [token],
|
||||
"[redacted transport credential]",
|
||||
),
|
||||
bytes: snapshot.stderr.observedBytes,
|
||||
|
||||
@@ -80,6 +80,18 @@ describe("IDA diagnostic credential redaction", () => {
|
||||
"raw=[redacted transport credential]; wire=[redacted transport credential]; token=[redacted transport credential]; proxy=[redacted transport credential]; bare=[redacted transport credential]; cookie=[redacted transport credential]",
|
||||
);
|
||||
});
|
||||
it("redacts a Set-Cookie value but not its attributes", () => {
|
||||
const redact = createIdaDiagnosticRedactor({
|
||||
url: "http://127.0.0.1:12345/mcp",
|
||||
headers: { "Set-Cookie": "sid=private-secret; Path=/; HttpOnly" },
|
||||
mode: "attached",
|
||||
protocol: "legacy",
|
||||
timeoutMs: 1000,
|
||||
}).text;
|
||||
expect(redact("rejected private-secret at /caller/input")).toBe(
|
||||
"rejected [redacted transport credential] at /caller/input",
|
||||
);
|
||||
});
|
||||
it("does not treat empty HTTP authentication as diagnostic content", () => {
|
||||
const redact = createIdaDiagnosticRedactor({
|
||||
url: "http://127.0.0.1:12345/mcp",
|
||||
|
||||
@@ -9,10 +9,11 @@ import {
|
||||
writeIdaDiagnostic,
|
||||
type IdaDiagnosticMatch,
|
||||
} from "./IdaDiagnosticWriter.js";
|
||||
import { idaExplicitSensitiveValues } from "./IdaRegistrationIdentity.js";
|
||||
import {
|
||||
isIdaAuthenticationHeader,
|
||||
idaExplicitSensitiveValues,
|
||||
} from "./IdaRegistrationIdentity.js";
|
||||
explicitValueSpans,
|
||||
isTransportCredentialHeader,
|
||||
} from "../domain/explicitSensitiveValues.js";
|
||||
import type { IdaConfiguration } from "./IdaConfiguration.js";
|
||||
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
|
||||
|
||||
@@ -155,58 +156,30 @@ export const createIdaDiagnosticRedactor = (
|
||||
? Object.entries(config.headers)
|
||||
: [];
|
||||
const values = headers.flatMap(([name, value]) => {
|
||||
if (!isIdaAuthenticationHeader(name)) return [];
|
||||
if (!isTransportCredentialHeader(name)) return [];
|
||||
// Fetch normalizes surrounding HTTP whitespace before sending headers.
|
||||
const normalized = value.replace(/^[ \t\r\n]+|[ \t\r\n]+$/gu, "");
|
||||
if (normalized.length === 0) return [];
|
||||
const field = name.toLowerCase();
|
||||
return [
|
||||
value,
|
||||
normalized,
|
||||
...(name.toLowerCase() === "cookie"
|
||||
...(field === "cookie"
|
||||
? cookiePayloads(normalized)
|
||||
: authorizationPayloads(normalized)),
|
||||
: field === "set-cookie"
|
||||
? // RFC 6265 4.1: only the leading cookie-pair carries the credential;
|
||||
// attributes such as Path=/ are not secrets.
|
||||
cookiePayloads(normalized.split(";", 1)[0] ?? "")
|
||||
: authorizationPayloads(normalized)),
|
||||
];
|
||||
});
|
||||
const secrets = [
|
||||
...new Set(
|
||||
[...values, ...idaExplicitSensitiveValues(config)].filter(
|
||||
(value) => value.length > 0,
|
||||
),
|
||||
),
|
||||
].sort((left, right) => right.length - left.length);
|
||||
const secrets = [...values, ...idaExplicitSensitiveValues(config)].filter(
|
||||
(value) => value.length > 0,
|
||||
);
|
||||
const replacement =
|
||||
secrets.length === 0 ? IDA_REDACTION_MARKER : redactionMarker(secrets);
|
||||
const alternatives = secrets
|
||||
.map((secret) => secret.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"))
|
||||
.join("|");
|
||||
// Look ahead at each source position; a global consuming match loses overlap.
|
||||
// Longest-first alternatives cover every shorter match at the same position.
|
||||
const pattern =
|
||||
alternatives.length === 0
|
||||
? undefined
|
||||
: new RegExp("(?=(" + alternatives + "))", "gu");
|
||||
const matches = function* (value: string): Iterable<IdaDiagnosticMatch> {
|
||||
if (pattern === undefined) return;
|
||||
let pending: IdaDiagnosticMatch | undefined;
|
||||
for (const match of value.matchAll(pattern)) {
|
||||
const credential = match[1];
|
||||
if (credential === undefined || credential.length === 0)
|
||||
throw new AnalysisProtocolError(
|
||||
"IDA diagnostic matching omitted its credential span.",
|
||||
);
|
||||
const span = { start: match.index, end: match.index + credential.length };
|
||||
if (pending !== undefined && span.start < pending.end) {
|
||||
pending = {
|
||||
start: pending.start,
|
||||
end: Math.max(pending.end, span.end),
|
||||
};
|
||||
} else {
|
||||
if (pending !== undefined) yield pending;
|
||||
pending = span;
|
||||
}
|
||||
}
|
||||
if (pending !== undefined) yield pending;
|
||||
};
|
||||
const matches = (value: string): Iterable<IdaDiagnosticMatch> =>
|
||||
explicitValueSpans(value, secrets);
|
||||
const text = boundedDiagnostic((message) =>
|
||||
writeIdaDiagnostic(message, matches(message), replacement),
|
||||
);
|
||||
|
||||
@@ -1,14 +1,9 @@
|
||||
import { digestCanonicalValue } from "../domain/canonicalDigest.js";
|
||||
import { isTransportCredentialHeader } from "../domain/explicitSensitiveValues.js";
|
||||
import type { IdaConfiguration } from "./IdaConfiguration.js";
|
||||
|
||||
/** HTTP authentication fields have protocol-defined credential meaning. */
|
||||
export const isIdaAuthenticationHeader = (name: string): boolean =>
|
||||
["authorization", "proxy-authorization", "cookie"].includes(
|
||||
name.toLowerCase(),
|
||||
);
|
||||
|
||||
const isSensitiveHeader = (config: IdaConfiguration, name: string): boolean =>
|
||||
isIdaAuthenticationHeader(name) ||
|
||||
isTransportCredentialHeader(name) ||
|
||||
(config.sensitive?.headers.some(
|
||||
(selected) => selected.toLowerCase() === name.toLowerCase(),
|
||||
) ??
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
import type { ProviderProcessSnapshot } from "./ProviderProcess.js";
|
||||
|
||||
/** Redact explicit transport credentials from complete and budget-cut output. */
|
||||
export const redactCapturedTransportCredential = (
|
||||
stream: Pick<
|
||||
ProviderProcessSnapshot["stdout"],
|
||||
"text" | "bytes" | "observedBytes"
|
||||
>,
|
||||
token: string | undefined,
|
||||
replacement: string,
|
||||
): string => {
|
||||
if (token === undefined || token.length === 0) return stream.text;
|
||||
|
||||
const { text } = stream;
|
||||
const cutOff = stream.observedBytes > stream.bytes;
|
||||
if (cutOff && !text.endsWith(token)) {
|
||||
const maximumPrefixLength = Math.min(text.length, token.length - 1);
|
||||
for (
|
||||
let prefixLength = maximumPrefixLength;
|
||||
prefixLength > 0;
|
||||
prefixLength -= 1
|
||||
) {
|
||||
if (text.endsWith(token.slice(0, prefixLength)))
|
||||
return `${text
|
||||
.slice(0, text.length - prefixLength)
|
||||
.replaceAll(token, replacement)}${replacement}`;
|
||||
}
|
||||
}
|
||||
|
||||
return text.replaceAll(token, replacement);
|
||||
};
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
ProviderProcessSupervisor,
|
||||
type ProcessDiagnosticBudget,
|
||||
} from "./ProviderProcess.js";
|
||||
import { redactCapturedTransportCredential } from "./ProviderDiagnosticRedaction.js";
|
||||
import { redactExplicitOutput } from "../domain/explicitSensitiveValues.js";
|
||||
|
||||
it("retains final stderr between verified exit and output close during stop", async () => {
|
||||
const producer = Object.assign(new EventEmitter(), {
|
||||
@@ -544,13 +544,9 @@ it.each(["stdout", "stderr"] as const)(
|
||||
observedBytes: Buffer.byteLength(token),
|
||||
text: token.slice(0, 8),
|
||||
});
|
||||
expect(
|
||||
redactCapturedTransportCredential(
|
||||
scoped.snapshot[stream],
|
||||
token,
|
||||
"[REDACTED]",
|
||||
),
|
||||
).toBe("[REDACTED]");
|
||||
expect(redactExplicitOutput(scoped.snapshot[stream], [token])).toBe(
|
||||
"[REDACTED]",
|
||||
);
|
||||
} finally {
|
||||
supervisor.dispose();
|
||||
producer.stdout.destroy();
|
||||
|
||||
Reference in new issue
Block a user