import { createHash } from "node:crypto"; import canonicalize from "canonicalize"; import type { InspectWebPageInput } from "../domain/browserObservation.js"; import type { WebPageInspection } from "../domain/browserObservationSchemas.js"; import { decodePaddedBase64 } from "../domain/paddedBase64.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { CdpConnection } from "./CdpConnection.js"; import { optionalCdpCommand } from "./CdpOptionalCommand.js"; import { recordValue, recordsValue, cdpStringValue, } from "./CdpCaptureValues.js"; type StorageFingerprint = WebPageInspection["storage"]["content_fingerprints"][number]; export interface CapturedStorageItems { readonly items: readonly { readonly key: string; readonly value: string; }[]; readonly complete: boolean; } export interface CapturedCache { readonly name: string; readonly id: string; } interface StorageFingerprintContext { readonly connection: CdpConnection; readonly sessionId: string | undefined; readonly input: InspectWebPageInput; readonly signal?: AbortSignal; } interface FingerprintState { readonly items: StorageFingerprint[]; complete: boolean; } interface FingerprintRun { readonly context: StorageFingerprintContext; readonly limitations: string[]; readonly state: FingerprintState; } interface CaptureStorageFingerprintInput { readonly context: StorageFingerprintContext; readonly origin: string; readonly cookieUrl: string; readonly local: CapturedStorageItems; readonly session: CapturedStorageItems; readonly indexedDbNames: readonly string[]; readonly caches: readonly CapturedCache[]; readonly limitations: string[]; } type StorageFingerprintCapture = { readonly items: readonly StorageFingerprint[]; readonly complete: boolean; }; /** Capture stable hashes of approved storage content without returning values. */ export const captureStorageFingerprints = async ({ context, origin, cookieUrl, local, session, indexedDbNames, caches, limitations, }: CaptureStorageFingerprintInput): Promise => { const state: FingerprintState = { items: [], complete: local.complete && session.complete, }; const run = { context, limitations, state }; addStorageItems(state, "local_storage", local.items); addStorageItems(state, "session_storage", session.items); await addCookies(run, cookieUrl); await addIndexedDb(run, origin, indexedDbNames); await addCaches(run, caches); state.items.sort((left, right) => compareUnicodeCodePoints( `${left.scope}:${left.identity_sha256}`, `${right.scope}:${right.identity_sha256}`, ), ); const complete = state.complete && state.items.every(({ complete }) => complete); return { items: state.items, complete }; }; const addStorageItems = ( state: FingerprintState, scope: "local_storage" | "session_storage", items: CapturedStorageItems["items"], ): void => { for (const item of items) addFingerprint(state, { scope, identity: item.key, value: item.value, complete: true, }); }; const addCookies = async ( run: FingerprintRun, pageUrl: string, ): Promise => { run.limitations.push( "Cookie fingerprints cover cookies applicable to the current main-frame URL, not every path on its origin.", ); const raw = await optionalCdpCommand( run.context, "Network.getCookies", { urls: [pageUrl] }, run.limitations, ); if (raw === undefined) { run.state.complete = false; return; } const cookieValues = recordValue(raw)?.cookies; if (!Array.isArray(cookieValues)) { run.state.complete = false; return; } const cookies = recordsValue(cookieValues); if (cookies.length !== cookieValues.length) run.state.complete = false; for (const cookie of cookies) { const name = cdpStringValue(cookie.name); const value = cdpStringValue(cookie.value); const domain = cdpStringValue(cookie.domain); const path = cdpStringValue(cookie.path); if ( name === undefined || value === undefined || domain === undefined || path === undefined ) { run.state.complete = false; continue; } addFingerprint(run.state, { scope: "cookie", identity: { domain, path, name, partition_key: cookie.partitionKey ?? null, }, value, complete: true, }); } }; const addIndexedDb = async ( run: FingerprintRun, origin: string, names: readonly string[], ): Promise => { for (const databaseName of names) { const raw = await optionalCdpCommand( run.context, "IndexedDB.requestDatabase", { securityOrigin: origin, databaseName }, run.limitations, ); const database = recordValue(recordValue(raw)?.databaseWithObjectStores); if (raw === undefined || database === undefined) { run.state.complete = false; continue; } if (cdpStringValue(database.name) !== databaseName) { run.state.complete = false; continue; } if (!Array.isArray(database.objectStores)) { run.state.complete = false; continue; } const stores = recordsValue(database.objectStores); if (stores.length !== database.objectStores.length) run.state.complete = false; addFingerprint(run.state, { scope: "indexed_db_schema", identity: databaseName, value: stores.map(indexedDbStoreSchema), complete: stores.length === database.objectStores.length && indexedDbSchemaComplete(stores), }); for (const store of stores) { const objectStoreName = cdpStringValue(store.name); if (objectStoreName === undefined) { run.state.complete = false; continue; } await addIndexedDbRecords(run, { origin, databaseName, objectStoreName, }); } } }; const indexedDbStoreSchema = (store: Readonly>) => ({ name: cdpStringValue(store.name) ?? null, key_path: stableValue(store.keyPath), auto_increment: store.autoIncrement === true, indexes: recordsValue(store.indexes).map((index) => ({ name: cdpStringValue(index.name) ?? null, key_path: stableValue(index.keyPath), unique: index.unique === true, multi_entry: index.multiEntry === true, })), }); const indexedDbSchemaComplete = ( stores: readonly Readonly>[], ): boolean => stores.every( (store) => cdpStringValue(store.name) !== undefined && indexedDbKeyPathComplete(store.keyPath) && typeof store.autoIncrement === "boolean" && Array.isArray(store.indexes) && recordsValue(store.indexes).length === store.indexes.length && recordsValue(store.indexes).every( (index) => cdpStringValue(index.name) !== undefined && indexedDbKeyPathComplete(index.keyPath) && typeof index.unique === "boolean" && typeof index.multiEntry === "boolean", ), ); const indexedDbKeyPathComplete = (value: unknown): boolean => { const keyPath = recordValue(value); if (keyPath === undefined) return false; switch (keyPath.type) { case "null": return true; case "string": return typeof keyPath.string === "string"; case "array": return ( Array.isArray(keyPath.array) && keyPath.array.every((key) => typeof key === "string") ); default: return false; } }; const addIndexedDbRecords = async ( run: FingerprintRun, input: { readonly origin: string; readonly databaseName: string; readonly objectStoreName: string; }, ): Promise => { let skipCount = 0; while (true) { const raw = await optionalCdpCommand( run.context, "IndexedDB.requestData", { securityOrigin: input.origin, databaseName: input.databaseName, objectStoreName: input.objectStoreName, skipCount, pageSize: STORAGE_ENTRY_PAGE_SIZE, }, run.limitations, ); const result = recordValue(raw); if (raw === undefined || result === undefined) { run.state.complete = false; return; } if ( !Array.isArray(result.objectStoreDataEntries) || typeof result.hasMore !== "boolean" ) { run.state.complete = false; return; } const entries = recordsValue(result.objectStoreDataEntries); if (entries.length !== result.objectStoreDataEntries.length) run.state.complete = false; for (const entry of entries) { const key = stableValue(entry.key); const primaryKey = stableValue(entry.primaryKey); // A missing remote object is an omission, not the JavaScript value // "undefined". Keep the unavailable digest null and coverage partial. const value = entry.value === undefined ? undefined : stableValue(entry.value); addFingerprint(run.state, { scope: "indexed_db_record", identity: { database: input.databaseName, store: input.objectStoreName, key, primaryKey, }, value, complete: remoteObjectComplete(entry.key) && remoteObjectComplete(entry.primaryKey) && remoteObjectComplete(entry.value), }); } if (result.hasMore !== true) return; if ( entries.length === 0 || skipCount > Number.MAX_SAFE_INTEGER - entries.length ) { run.state.complete = false; return; } skipCount += entries.length; } }; const addCaches = async ( run: FingerprintRun, caches: readonly CapturedCache[], ): Promise => { for (const cache of caches) { let skipCount = 0; let returnCount: number | undefined; while (returnCount === undefined || skipCount < returnCount) { const raw = await optionalCdpCommand( run.context, "CacheStorage.requestEntries", { cacheId: cache.id, skipCount, pageSize: STORAGE_ENTRY_PAGE_SIZE }, run.limitations, ); const result = recordValue(raw); if (raw === undefined || result === undefined) { run.state.complete = false; break; } if ( !Array.isArray(result.cacheDataEntries) || typeof result.returnCount !== "number" || !Number.isSafeInteger(result.returnCount) || result.returnCount < 0 ) { run.state.complete = false; break; } const entries = recordsValue(result.cacheDataEntries); if (entries.length !== result.cacheDataEntries.length) run.state.complete = false; for (const entry of entries) await addCacheEntry(run, cache, entry); returnCount = result.returnCount; if (skipCount + entries.length >= returnCount) break; if ( entries.length === 0 || skipCount > Number.MAX_SAFE_INTEGER - entries.length ) { run.state.complete = false; break; } skipCount += entries.length; } } }; const addCacheEntry = async ( run: FingerprintRun, cache: CapturedCache, entry: Readonly>, ): Promise => { const requestURL = cdpStringValue(entry.requestURL); const requestMethod = cdpStringValue(entry.requestMethod); if (requestURL === undefined || requestMethod === undefined) { run.state.complete = false; return; } if (!Array.isArray(entry.requestHeaders)) { run.state.complete = false; return; } const headerRecords = recordsValue(entry.requestHeaders); const requestHeaders = headerRecords.flatMap((header) => { const name = cdpStringValue(header.name); const value = cdpStringValue(header.value); return name === undefined || value === undefined ? [] : [{ name, value }]; }); if ( headerRecords.length !== entry.requestHeaders.length || requestHeaders.length !== headerRecords.length ) { run.state.complete = false; return; } const raw = await optionalCdpCommand( run.context, "CacheStorage.requestCachedResponse", { cacheId: cache.id, requestURL, requestHeaders }, run.limitations, ); const body = cdpStringValue(recordValue(recordValue(raw)?.response)?.body); const decoded = body === undefined ? undefined : decodePaddedBase64(body); const complete = decoded !== undefined; if (!complete) run.state.complete = false; addFingerprint(run.state, { scope: "cache_entry", identity: { cache: cache.name, requestURL, requestMethod, requestHeaders }, value: decoded, complete, }); }; const addFingerprint = ( state: FingerprintState, input: { readonly scope: StorageFingerprint["scope"]; readonly identity: unknown; readonly value: unknown; readonly complete: boolean; }, ): void => { state.items.push({ scope: input.scope, identity_sha256: digest(input.identity), value_sha256: input.value === undefined ? null : digest(input.value), complete: input.complete && stableValueComplete(input.identity) && stableValueComplete(input.value), }); }; const STORAGE_ENTRY_PAGE_SIZE = 1_000; const digest = (value: unknown): string => createHash("sha256").update(encodedValue(value)).digest("hex"); const encodedValue = (value: unknown): Uint8Array | string => { if (value instanceof Uint8Array) return value; const encoded = canonicalize(stableValue(value)); if (encoded === undefined) throw new TypeError("Expected canonical JSON"); return encoded; }; const stableValue = (value: unknown): unknown => { if ( value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean" ) return value; if (Array.isArray(value)) return value.map((item) => stableValue(item)); const record = recordValue(value); if (record === undefined) return String(value); return Object.fromEntries( Object.entries(record) .filter(([key]) => key !== "objectId" && key !== "customPreview") .sort(([left], [right]) => compareUnicodeCodePoints(left, right)) .map(([key, item]) => [key, stableValue(item)]), ); }; const stableValueComplete = (value: unknown): boolean => { if (value instanceof Uint8Array) return true; if ( value === null || value === undefined || typeof value === "string" || typeof value === "number" || typeof value === "boolean" ) return true; if (Array.isArray(value)) return value.every((item) => stableValueComplete(item)); const record = recordValue(value); if (record === undefined) return false; const entries = Object.entries(record).filter( ([key]) => key !== "objectId" && key !== "customPreview", ); return entries.every(([, item]) => stableValueComplete(item)); }; const remoteObjectComplete = (value: unknown): boolean => { const record = recordValue(value); if ( record === undefined || record.objectId !== undefined || record.preview !== undefined || record.customPreview !== undefined || !stableValueComplete(record) ) return false; switch (record.type) { case "undefined": return ( record.value === undefined && record.unserializableValue === undefined ); case "string": case "boolean": return ( typeof record.value === record.type && record.unserializableValue === undefined ); case "number": return record.unserializableValue === undefined ? typeof record.value === "number" && Number.isFinite(record.value) : record.value === undefined && typeof record.unserializableValue === "string" && ["-0", "NaN", "Infinity", "-Infinity"].includes( record.unserializableValue, ); case "bigint": return ( record.value === undefined && typeof record.unserializableValue === "string" && /^-?\d+n$/u.test(record.unserializableValue) ); case "object": return ( record.unserializableValue === undefined && (record.subtype === "null" ? record.value === null : record.value !== null && typeof record.value === "object") ); default: return false; } };