Files
reverse-engineering-anything/src/browser/CdpCaptureStorageFingerprints.ts
T
morluto b51ed4c3ec refactor(kernel): migrate the remaining SHA-256 hex digests to sha256Hex and move isRecord into src/kernel (#11571)
* refactor(kernel): migrate the remaining one-shot SHA-256 hex digests to sha256Hex

* refactor(kernel): move isRecord into src/kernel and replace its inline copies (#11574)

* fix(scripts): import isRecord from src/kernel in the managed manifest parser
2026-11-15 20:53:36 -05:00

540 lines
16 KiB
TypeScript

import canonicalize from "canonicalize";
import type { InspectWebPageInput } from "../domain/browserObservation.js";
import type { WebPageInspection } from "../domain/browserObservationSchemas.js";
import { decodePaddedBase64 } from "../domain/paddedBase64.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { CdpConnection } from "./CdpConnection.js";
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
import {
recordValue,
recordsValue,
cdpStringValue,
} from "./CdpCaptureValues.js";
import { sha256Hex } from "../kernel/sha256Hex.js";
type StorageFingerprint =
WebPageInspection["storage"]["content_fingerprints"][number];
export interface CapturedStorageItems {
readonly items: readonly {
readonly key: string;
readonly value: string | undefined;
}[];
readonly complete: boolean;
}
export interface CapturedCache {
readonly name: string;
readonly id: string | undefined;
}
interface StorageFingerprintContext {
readonly connection: CdpConnection;
readonly sessionId: string | undefined;
readonly input: InspectWebPageInput;
readonly signal?: AbortSignal;
}
interface FingerprintState {
readonly items: StorageFingerprint[];
complete: boolean;
}
interface FingerprintRun {
readonly context: StorageFingerprintContext;
readonly limitations: string[];
readonly state: FingerprintState;
}
interface CaptureStorageFingerprintInput {
readonly context: StorageFingerprintContext;
readonly origin: string;
readonly cookieUrl: string;
readonly local: CapturedStorageItems;
readonly session: CapturedStorageItems;
readonly indexedDbNames: readonly string[];
readonly caches: readonly CapturedCache[];
readonly limitations: string[];
}
type StorageFingerprintCapture = {
readonly items: readonly StorageFingerprint[];
readonly complete: boolean;
};
/** Capture stable hashes of approved storage content without returning values. */
export const captureStorageFingerprints = async ({
context,
origin,
cookieUrl,
local,
session,
indexedDbNames,
caches,
limitations,
}: CaptureStorageFingerprintInput): Promise<StorageFingerprintCapture> => {
const state: FingerprintState = {
items: [],
complete: local.complete && session.complete,
};
const run = { context, limitations, state };
addStorageItems(state, "local_storage", local.items);
addStorageItems(state, "session_storage", session.items);
await addCookies(run, cookieUrl);
await addIndexedDb(run, origin, indexedDbNames);
await addCaches(run, caches);
state.items.sort((left, right) =>
compareUnicodeCodePoints(
`${left.scope}:${left.identity_sha256}`,
`${right.scope}:${right.identity_sha256}`,
),
);
const complete =
state.complete && state.items.every(({ complete }) => complete);
return { items: state.items, complete };
};
const addStorageItems = (
state: FingerprintState,
scope: "local_storage" | "session_storage",
items: CapturedStorageItems["items"],
): void => {
for (const item of items)
addFingerprint(state, {
scope,
identity: item.key,
value: item.value,
complete: item.value !== undefined,
});
};
const addCookies = async (
run: FingerprintRun,
pageUrl: string,
): Promise<void> => {
run.limitations.push(
"Cookie fingerprints cover cookies applicable to the current main-frame URL, not every path on its origin.",
);
const reply = await optionalCdpCommand(
run.context,
"Network.getCookies",
{ urls: [pageUrl] },
run.limitations,
);
if (!reply.ok) {
run.state.complete = false;
return;
}
const cookieValues = reply.value.cookies;
const cookies = recordsValue(cookieValues);
if (cookies.length !== cookieValues.length) run.state.complete = false;
for (const cookie of cookies) {
const name = cdpStringValue(cookie.name);
const value = cdpStringValue(cookie.value);
const domain = cdpStringValue(cookie.domain);
const path = cdpStringValue(cookie.path);
if (name === undefined || domain === undefined || path === undefined) {
run.state.complete = false;
continue;
}
// CDP Cookie.partitionKeyOpaque means the partition exists but is not
// serializable. A missing partitionKey means unpartitioned, so hashing
// that absence would align an unresolved partition with a different cookie.
// https://chromedevtools.github.io/devtools-protocol/tot/Network/#type-Cookie
const partitionOpaque = cookie.partitionKeyOpaque === true;
addFingerprint(run.state, {
scope: "cookie",
identity: {
domain,
path,
name,
partition_key: cookie.partitionKey ?? null,
...(partitionOpaque ? { partition_opaque: true } : {}),
},
value,
complete: value !== undefined && !partitionOpaque,
});
}
};
const addIndexedDb = async (
run: FingerprintRun,
origin: string,
names: readonly string[],
): Promise<void> => {
for (const databaseName of names) {
const reply = await optionalCdpCommand(
run.context,
"IndexedDB.requestDatabase",
{ securityOrigin: origin, databaseName },
run.limitations,
);
if (!reply.ok) {
run.state.complete = false;
continue;
}
const database = reply.value.databaseWithObjectStores;
if (database.name !== databaseName) {
run.state.complete = false;
continue;
}
const stores = recordsValue(database.objectStores);
if (stores.length !== database.objectStores.length)
run.state.complete = false;
const { version } = database;
if (version === undefined) {
const limitation =
"An IndexedDB schema fingerprint is incomplete because its database version is unavailable, invalid, or outside exact integer precision.";
if (!run.limitations.includes(limitation))
run.limitations.push(limitation);
}
addFingerprint(run.state, {
scope: "indexed_db_schema",
identity: databaseName,
value: {
version: version ?? null,
object_stores: stores.map(indexedDbStoreSchema),
},
complete:
version !== undefined &&
stores.length === database.objectStores.length &&
indexedDbSchemaComplete(stores),
});
for (const store of stores) {
const objectStoreName = cdpStringValue(store.name);
if (objectStoreName === undefined) {
run.state.complete = false;
continue;
}
await addIndexedDbRecords(run, {
origin,
databaseName,
objectStoreName,
});
}
}
};
const indexedDbStoreSchema = (store: Readonly<Record<string, unknown>>) => ({
name: cdpStringValue(store.name) ?? null,
key_path: stableValue(store.keyPath),
auto_increment: store.autoIncrement === true,
indexes: recordsValue(store.indexes).map((index) => ({
name: cdpStringValue(index.name) ?? null,
key_path: stableValue(index.keyPath),
unique: index.unique === true,
multi_entry: index.multiEntry === true,
})),
});
const indexedDbSchemaComplete = (
stores: readonly Readonly<Record<string, unknown>>[],
): boolean =>
stores.every(
(store) =>
cdpStringValue(store.name) !== undefined &&
indexedDbKeyPathComplete(store.keyPath) &&
typeof store.autoIncrement === "boolean" &&
Array.isArray(store.indexes) &&
recordsValue(store.indexes).length === store.indexes.length &&
recordsValue(store.indexes).every(
(index) =>
cdpStringValue(index.name) !== undefined &&
indexedDbKeyPathComplete(index.keyPath) &&
typeof index.unique === "boolean" &&
typeof index.multiEntry === "boolean",
),
);
const indexedDbKeyPathComplete = (value: unknown): boolean => {
const keyPath = recordValue(value);
if (keyPath === undefined) return false;
switch (keyPath.type) {
case "null":
return true;
case "string":
return typeof keyPath.string === "string";
case "array":
return (
Array.isArray(keyPath.array) &&
keyPath.array.every((key) => typeof key === "string")
);
default:
return false;
}
};
const addIndexedDbRecords = async (
run: FingerprintRun,
input: {
readonly origin: string;
readonly databaseName: string;
readonly objectStoreName: string;
},
): Promise<void> => {
let skipCount = 0;
while (true) {
const reply = await optionalCdpCommand(
run.context,
"IndexedDB.requestData",
{
securityOrigin: input.origin,
databaseName: input.databaseName,
objectStoreName: input.objectStoreName,
skipCount,
pageSize: STORAGE_ENTRY_PAGE_SIZE,
},
run.limitations,
);
if (!reply.ok) {
run.state.complete = false;
return;
}
const result = reply.value;
const examinedCount = result.objectStoreDataEntries.length;
const entries = recordsValue(result.objectStoreDataEntries);
if (entries.length !== result.objectStoreDataEntries.length)
run.state.complete = false;
for (const entry of entries) {
const key = decodeRemoteObject(entry.key);
const primaryKey = decodeRemoteObject(entry.primaryKey);
const value = decodeRemoteObject(entry.value);
addFingerprint(run.state, {
scope: "indexed_db_record",
identity: {
database: input.databaseName,
store: input.objectStoreName,
key,
primaryKey,
},
value,
complete:
key !== undefined && primaryKey !== undefined && value !== undefined,
});
}
if (!result.hasMore) return;
if (
examinedCount === 0 ||
skipCount > Number.MAX_SAFE_INTEGER - examinedCount
) {
run.state.complete = false;
return;
}
skipCount += examinedCount;
}
};
const addCaches = async (
run: FingerprintRun,
caches: readonly CapturedCache[],
): Promise<void> => {
for (const cache of caches) {
const cacheId = cache.id;
if (cacheId === undefined) {
run.state.complete = false;
continue;
}
let skipCount = 0;
let returnCount: number | undefined;
while (returnCount === undefined || skipCount < returnCount) {
const reply = await optionalCdpCommand(
run.context,
"CacheStorage.requestEntries",
{ cacheId, skipCount, pageSize: STORAGE_ENTRY_PAGE_SIZE },
run.limitations,
);
if (!reply.ok) {
run.state.complete = false;
break;
}
const result = reply.value;
const examinedCount = result.cacheDataEntries.length;
const entries = recordsValue(result.cacheDataEntries);
if (entries.length !== result.cacheDataEntries.length)
run.state.complete = false;
for (const entry of entries)
await addCacheEntry(run, { name: cache.name, id: cacheId }, entry);
returnCount = result.returnCount;
if (skipCount + examinedCount >= returnCount) break;
if (
examinedCount === 0 ||
skipCount > Number.MAX_SAFE_INTEGER - examinedCount
) {
run.state.complete = false;
break;
}
skipCount += examinedCount;
}
}
};
const addCacheEntry = async (
run: FingerprintRun,
cache: { readonly name: string; readonly id: string },
entry: Readonly<Record<string, unknown>>,
): Promise<void> => {
const requestURL = cdpStringValue(entry.requestURL);
const requestMethod = cdpStringValue(entry.requestMethod);
if (requestURL === undefined || requestMethod === undefined) {
run.state.complete = false;
return;
}
if (!Array.isArray(entry.requestHeaders)) {
run.state.complete = false;
return;
}
const headerRecords = recordsValue(entry.requestHeaders);
const requestHeaders = headerRecords.flatMap((header) => {
const name = cdpStringValue(header.name);
const value = cdpStringValue(header.value);
return name === undefined || value === undefined ? [] : [{ name, value }];
});
if (
headerRecords.length !== entry.requestHeaders.length ||
requestHeaders.length !== headerRecords.length
) {
run.state.complete = false;
return;
}
const reply = await optionalCdpCommand(
run.context,
"CacheStorage.requestCachedResponse",
{ cacheId: cache.id, requestURL, requestHeaders },
run.limitations,
);
const decoded = reply.ok
? decodePaddedBase64(reply.value.response.body)
: undefined;
const complete = decoded !== undefined;
if (!complete) run.state.complete = false;
addFingerprint(run.state, {
scope: "cache_entry",
identity: { cache: cache.name, requestURL, requestMethod, requestHeaders },
value: decoded,
complete,
});
};
const addFingerprint = (
state: FingerprintState,
input: {
readonly scope: StorageFingerprint["scope"];
readonly identity: unknown;
readonly value: unknown;
readonly complete: boolean;
},
): void => {
const fingerprint = {
identity_sha256: digest(input.identity),
value_sha256: input.value === undefined ? null : digest(input.value),
complete:
input.complete &&
stableValueComplete(input.identity) &&
stableValueComplete(input.value),
};
// The IndexedDB schema projection includes the database version.
state.items.push(
input.scope === "indexed_db_schema"
? { ...fingerprint, scope: input.scope, projection_version: 2 }
: { ...fingerprint, scope: input.scope },
);
};
const STORAGE_ENTRY_PAGE_SIZE = 1_000;
const digest = (value: unknown): string => sha256Hex(encodedValue(value));
const encodedValue = (value: unknown): Uint8Array | string => {
if (value instanceof Uint8Array) return value;
const encoded = canonicalize(stableValue(value));
if (encoded === undefined) throw new TypeError("Expected canonical JSON");
return encoded;
};
const stableValue = (value: unknown): unknown => {
if (
value === null ||
typeof value === "string" ||
typeof value === "number" ||
typeof value === "boolean"
)
return value;
if (Array.isArray(value)) return value.map((item) => stableValue(item));
const record = recordValue(value);
if (record === undefined) return String(value);
return Object.fromEntries(
Object.entries(record)
.sort(([left], [right]) => compareUnicodeCodePoints(left, right))
.map(([key, item]) => [key, stableValue(item)]),
);
};
const stableValueComplete = (value: unknown): boolean => {
if (value instanceof Uint8Array) return true;
if (
value === null ||
value === undefined ||
typeof value === "string" ||
typeof value === "number" ||
typeof value === "boolean"
)
return true;
if (Array.isArray(value))
return value.every((item) => stableValueComplete(item));
const record = recordValue(value);
if (record === undefined) return false;
const entries = Object.entries(record);
return entries.every(([, item]) => stableValueComplete(item));
};
/** Decode CDP value evidence once; debugger labels/references are not content.
* https://chromedevtools.github.io/devtools-protocol/tot/Runtime/#type-RemoteObject
*/
const decodeRemoteObject = (value: unknown): unknown => {
const record = recordValue(value);
if (record === undefined) return undefined;
const { type, unserializableValue } = record;
const serialized = record.value;
const primitive = (valid: boolean): unknown =>
valid ? { type, value: serialized } : undefined;
switch (type) {
case "undefined":
return serialized === undefined && unserializableValue === undefined
? { type }
: undefined;
case "string":
case "boolean":
return primitive(
typeof serialized === type && unserializableValue === undefined,
);
case "number":
if (unserializableValue === undefined)
return primitive(
typeof serialized === "number" && Number.isFinite(serialized),
);
return serialized === undefined &&
typeof unserializableValue === "string" &&
["-0", "NaN", "Infinity", "-Infinity"].includes(unserializableValue)
? { type, unserializableValue }
: undefined;
case "bigint":
return serialized === undefined &&
typeof unserializableValue === "string" &&
/^-?\d+n$/u.test(unserializableValue)
? { type, unserializableValue }
: undefined;
case "object":
return primitive(
unserializableValue === undefined &&
(record.subtype === "null"
? serialized === null
: serialized !== null &&
typeof serialized === "object" &&
stableValueComplete(serialized)),
);
default:
return undefined;
}
};