mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(captures): preserve relative redirect URL evidence
This commit is contained in:
1 parent
2012efe5fd
commit
99d4ea9c05
7 files changed
+208
-22
No files matched your search
@@ -26,7 +26,7 @@ it.each([
|
||||
"excludes URL userinfo after optional whitespace while preserving the original header text: $value",
|
||||
({ value, expected }) => {
|
||||
const redactor = new CaptureRedaction([]);
|
||||
expect(redactor.text(value, "/response/headers/0/value", true)).toBe(
|
||||
expect(redactor.text(value, "/response/headers/0/value", {})).toBe(
|
||||
expected,
|
||||
);
|
||||
expect(redactor.redactions).toEqual([
|
||||
@@ -56,7 +56,7 @@ it.each([
|
||||
"/\t/user:password@example.test/path",
|
||||
])("excludes credentials whose raw URL span is ambiguous: %s", (value) => {
|
||||
const redactor = new CaptureRedaction([]);
|
||||
expect(redactor.text(value, "/request/url", true)).toBeNull();
|
||||
expect(redactor.text(value, "/request/url", {})).toBeNull();
|
||||
expect(redactor.redactions).toEqual([
|
||||
{ pointer: "/request/url", reason: "transport-credential" },
|
||||
]);
|
||||
@@ -66,7 +66,7 @@ it.each([
|
||||
it("uses non-special URL authority rules to remove the complete userinfo span", () => {
|
||||
const value = "custom://user:secret@part\\more:password@host/path";
|
||||
const redactor = new CaptureRedaction([]);
|
||||
expect(redactor.text(value, "/response/redirectURL", true)).toBe(
|
||||
expect(redactor.text(value, "/response/redirectURL", {})).toBe(
|
||||
"custom://host/path",
|
||||
);
|
||||
expect(redactor.redactions).toEqual([
|
||||
@@ -83,7 +83,7 @@ it("preserves noncanonical URLs and path/query userinfo lookalikes without parse
|
||||
"//invalid host\\user:password@literal",
|
||||
]) {
|
||||
const redactor = new CaptureRedaction([]);
|
||||
expect(redactor.text(value, "/request/url", true)).toBe(value);
|
||||
expect(redactor.text(value, "/request/url", {})).toBe(value);
|
||||
expect(redactor.redactions).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js"
|
||||
import { parseRecordedTransportUrl } from "../../domain/recordedTransportUrl.js";
|
||||
|
||||
type Redaction = WebNetworkCaptureRecord["redactions"][number];
|
||||
type TransportUrlContext = { readonly credentialBaseUrl?: string };
|
||||
const CREDENTIAL_HEADERS = new Set([
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
@@ -39,7 +40,17 @@ export const isCaptureCredentialHeaderBytes = (
|
||||
export class CaptureRedaction {
|
||||
readonly redactions: Redaction[] = [];
|
||||
readonly #sensitiveByteValues: readonly Buffer[];
|
||||
constructor(readonly sensitiveValues: readonly string[]) {
|
||||
readonly #requestUrl: string | undefined;
|
||||
constructor(
|
||||
readonly sensitiveValues: readonly string[],
|
||||
source?: unknown,
|
||||
) {
|
||||
const request =
|
||||
captureObject(source) && captureObject(source.request)
|
||||
? source.request
|
||||
: undefined;
|
||||
this.#requestUrl =
|
||||
typeof request?.url === "string" ? request.url : undefined;
|
||||
// An unpaired surrogate has no UTF-8 representation. Buffer.from would
|
||||
// invent replacement-character bytes and incorrectly discard real evidence.
|
||||
this.#sensitiveByteValues = sensitiveValues.flatMap((value) => {
|
||||
@@ -49,10 +60,14 @@ export class CaptureRedaction {
|
||||
}
|
||||
|
||||
/** Preserve exact text unless explicitly sensitive or known transport URL userinfo. */
|
||||
text(value: string, pointer: string, transportUrl = false): string | null {
|
||||
text(
|
||||
value: string,
|
||||
pointer: string,
|
||||
transportUrl?: TransportUrlContext,
|
||||
): string | null {
|
||||
let result = value;
|
||||
if (transportUrl) {
|
||||
const { credentials } = parseRecordedTransportUrl(value);
|
||||
if (transportUrl !== undefined) {
|
||||
const { credentials } = parseRecordedTransportUrl(value, transportUrl);
|
||||
if (credentials === "ambiguous") return this.credential(pointer);
|
||||
if (credentials !== null) {
|
||||
result = `${value.slice(0, credentials.start)}${value.slice(credentials.end)}`;
|
||||
@@ -109,14 +124,21 @@ export class CaptureRedaction {
|
||||
harTransportUrl(
|
||||
pointer: string,
|
||||
parent?: Readonly<Record<string, unknown>>,
|
||||
): boolean {
|
||||
return (
|
||||
pointer === "/request/url" ||
|
||||
pointer === "/response/redirectURL" ||
|
||||
(/^\/(request|response)\/headers\/\d+\/value$/.test(pointer) &&
|
||||
typeof parent?.name === "string" &&
|
||||
["location", "referer", "origin"].includes(parent.name.toLowerCase()))
|
||||
);
|
||||
): TransportUrlContext | undefined {
|
||||
const header = /^\/(request|response)\/headers\/\d+\/value$/.test(pointer)
|
||||
? typeof parent?.name === "string"
|
||||
? parent.name.toLowerCase()
|
||||
: undefined
|
||||
: undefined;
|
||||
if (pointer === "/response/redirectURL" || header === "location")
|
||||
return this.#requestUrl === undefined
|
||||
? {}
|
||||
: { credentialBaseUrl: this.#requestUrl };
|
||||
return pointer === "/request/url" ||
|
||||
header === "referer" ||
|
||||
header === "origin"
|
||||
? {}
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Preserve bytes unless an explicitly declared UTF-8 literal occurs in them. */
|
||||
|
||||
@@ -109,7 +109,10 @@ const projectHar = (
|
||||
scope: "validation" | "container" | "entry",
|
||||
): Projection => {
|
||||
const redact = scope !== "validation";
|
||||
const redactor = new CaptureRedaction(sensitiveValues);
|
||||
const redactor = new CaptureRedaction(
|
||||
sensitiveValues,
|
||||
scope === "entry" ? value : undefined,
|
||||
);
|
||||
const numeric_literals: Projection["numeric_literals"] = [];
|
||||
const binary_fields: Projection["binary_fields"] = [];
|
||||
const projectNumber = (item: LosslessNumber, pointer: string): JsonValue => {
|
||||
@@ -192,7 +195,9 @@ const projectHar = (
|
||||
? redactor.text(
|
||||
item,
|
||||
pointer,
|
||||
scope === "entry" && redactor.harTransportUrl(pointer, parent),
|
||||
scope === "entry"
|
||||
? redactor.harTransportUrl(pointer, parent)
|
||||
: undefined,
|
||||
)
|
||||
: item;
|
||||
}
|
||||
|
||||
@@ -455,7 +455,9 @@ const project = (
|
||||
header &&
|
||||
["location", "referer", "origin"].includes(
|
||||
String(parent?.name).toLowerCase(),
|
||||
),
|
||||
)
|
||||
? {}
|
||||
: undefined,
|
||||
);
|
||||
if (item === null || typeof item === "boolean") return item;
|
||||
if (Array.isArray(item))
|
||||
|
||||
@@ -19,7 +19,7 @@ export const projectSazRecord = (
|
||||
source: SazValue,
|
||||
sensitiveValues: readonly string[],
|
||||
) => {
|
||||
const redactor = new CaptureRedaction(sensitiveValues);
|
||||
const redactor = new CaptureRedaction(sensitiveValues, source);
|
||||
const binary_fields: WebNetworkCaptureRecord["binary_fields"] = [];
|
||||
const visit = (
|
||||
value: SazValue,
|
||||
|
||||
@@ -17,15 +17,31 @@ export interface RecordedTransportUrl {
|
||||
*/
|
||||
export const parseRecordedTransportUrl = (
|
||||
value: string,
|
||||
options: { readonly includeQuery?: boolean } = {},
|
||||
options: {
|
||||
readonly includeQuery?: boolean;
|
||||
/** Recorded request context for credential interpretation, never projected identity. */
|
||||
readonly credentialBaseUrl?: string;
|
||||
} = {},
|
||||
): RecordedTransportUrl => {
|
||||
const prefix = /^[\x00-\x20]*(?:([A-Za-z][A-Za-z0-9+.-]*):)?\/\//u.exec(
|
||||
value,
|
||||
);
|
||||
const absolute = URL.parse(value);
|
||||
const credentialBase =
|
||||
options.credentialBaseUrl === undefined
|
||||
? null
|
||||
: URL.parse(options.credentialBaseUrl);
|
||||
if (credentialBase !== null) {
|
||||
// Inherited request authentication is not part of a relative header's text.
|
||||
credentialBase.username = "";
|
||||
credentialBase.password = "";
|
||||
}
|
||||
// Resolve references only to recognize credentials (including WHATWG's
|
||||
// backslash/control-character spellings), never to supply a projected origin.
|
||||
const parsed = absolute ?? URL.parse(value, "https://rea.invalid");
|
||||
const parsed =
|
||||
credentialBase === null
|
||||
? (absolute ?? URL.parse(value, "https://rea.invalid"))
|
||||
: URL.parse(value, credentialBase);
|
||||
const hasCredentials =
|
||||
parsed !== null && (parsed.username !== "" || parsed.password !== "");
|
||||
const origin =
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js";
|
||||
import { expect } from "vitest";
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { webNetworkCaptureSchema } from "../../../src/domain/webNetworkCapture.js";
|
||||
import { historicalHar } from "../../fixtures/historicalHar.js";
|
||||
import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
|
||||
// Chrome 156 / Playwright 1.58.2 follows a real HTTP 302 with this Location
|
||||
// to /next@name and retains the exact header in its exported HAR.
|
||||
// SAZ equivalents below are authored HTTP-message goldens, not Fiddler exports.
|
||||
const redirects = [
|
||||
{
|
||||
request: "http://example.test/start",
|
||||
location: "http:next@name",
|
||||
retained: "http:next@name",
|
||||
resolved: "http://example.test/next@name",
|
||||
retainedRedirect: "http://example.test/next@name",
|
||||
},
|
||||
{
|
||||
request: "http://source-user:source-pass@example.test/start",
|
||||
location: "http:next@name",
|
||||
retained: "http:next@name",
|
||||
// Some producers retain the Location spelling in redirectURL as well.
|
||||
resolved: "http:next@name",
|
||||
retainedRedirect: "http:next@name",
|
||||
},
|
||||
{
|
||||
request: "http://example.test/start",
|
||||
location: "http://redirect-user:redirect-pass@example.test/final",
|
||||
retained: "http://example.test/final",
|
||||
resolved: "http://redirect-user:redirect-pass@example.test/final",
|
||||
retainedRedirect: "http://example.test/final",
|
||||
},
|
||||
{
|
||||
request: "http://example.test/start",
|
||||
location: "//redirect-user:redirect-pass@example.test/final",
|
||||
retained: "//example.test/final",
|
||||
resolved: "http://redirect-user:redirect-pass@example.test/final",
|
||||
retainedRedirect: "http://example.test/final",
|
||||
},
|
||||
];
|
||||
|
||||
const harBytes = () => {
|
||||
const fixture = historicalHar();
|
||||
const entry = fixture.log.entries[0]!;
|
||||
return Buffer.from(
|
||||
JSON.stringify({
|
||||
log: {
|
||||
...fixture.log,
|
||||
entries: redirects.map(({ request, location, resolved }) => ({
|
||||
...entry,
|
||||
request: { ...entry.request, url: request },
|
||||
response: {
|
||||
...entry.response,
|
||||
status: 302,
|
||||
statusText: "Found",
|
||||
headers: [
|
||||
{ name: "Location", value: location },
|
||||
{ name: "X-Observed", value: "next@name" },
|
||||
],
|
||||
redirectURL: resolved,
|
||||
},
|
||||
})),
|
||||
},
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const sazBytes = async () => {
|
||||
const writer = new ZipWriter(new Uint8ArrayWriter(), {
|
||||
useWebWorkers: false,
|
||||
});
|
||||
for (const [ordinal, { request, location }] of redirects.entries()) {
|
||||
const messages = [
|
||||
["c", `GET ${request} HTTP/1.1\r\n\r\n`],
|
||||
[
|
||||
"s",
|
||||
`HTTP/1.1 302 Found\r\nLocation:${location}\r\nX-Observed:next@name\r\n\r\n`,
|
||||
],
|
||||
] as const;
|
||||
for (const [side, text] of messages)
|
||||
await writer.add(
|
||||
`raw/${ordinal + 1}_${side}.txt`,
|
||||
new Uint8ArrayReader(Buffer.from(text)),
|
||||
);
|
||||
}
|
||||
return Buffer.from(await writer.close());
|
||||
};
|
||||
|
||||
cliTest(
|
||||
"preserves same-scheme Location paths and excludes actual redirect credentials through CLI and MCP",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-history-redirects-");
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
for (const format of ["har", "saz"] as const) {
|
||||
const path = join(root, `redirects.${format}`);
|
||||
await writeFile(path, format === "har" ? harBytes() : await sazBytes());
|
||||
const output = await cli.run({
|
||||
arguments: ["inspect-web-network-capture", path, format, "--json"],
|
||||
});
|
||||
expect(output.exitCode, output.stderr).toBe(0);
|
||||
const evidence = parseEvidence(output.json);
|
||||
const capture = webNetworkCaptureSchema.parse(evidence.normalized_result);
|
||||
expect(capture.total_records).toBe(redirects.length);
|
||||
for (const [
|
||||
ordinal,
|
||||
{ retained, retainedRedirect },
|
||||
] of redirects.entries()) {
|
||||
const record = capture.records[ordinal]!;
|
||||
expect(record.reported).toMatchObject({
|
||||
request: { url: "http://example.test/start" },
|
||||
response: {
|
||||
...(format === "har" ? { redirectURL: retainedRedirect } : {}),
|
||||
headers: [
|
||||
{ name: "Location", value: retained },
|
||||
{ name: "X-Observed", value: "next@name" },
|
||||
],
|
||||
},
|
||||
});
|
||||
if (ordinal < 2)
|
||||
expect(record.redactions).not.toContainEqual({
|
||||
pointer: "/response/headers/0/value",
|
||||
reason: "transport-credential",
|
||||
});
|
||||
}
|
||||
expect(JSON.stringify(capture)).not.toMatch(/source-pass|redirect-pass/u);
|
||||
const reply = await call("inspect_web_network_capture", {
|
||||
capture_path: path,
|
||||
format,
|
||||
});
|
||||
expect(reply.isError, JSON.stringify(reply)).not.toBe(true);
|
||||
expect(parseEvidence(reply.structuredContent).evidence_id).toBe(
|
||||
evidence.evidence_id,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user