From 05b392dfd7b5b4e41d012b236f23400cc8576d93 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:54:55 +0800 Subject: [PATCH] fix(environment): admit serialized exports and scan HAR form fallbacks --- .../environment/environmentBuild.test.ts | 2 +- .../environment/environmentBuild.ts | 19 +------ .../environmentBuildReport.test.ts | 53 ++++++++++------- .../environment/environmentBuildReport.ts | 5 +- .../environment/environmentEvidenceBytes.ts | 19 +++++++ .../environment/environmentExport.ts | 23 ++++++++ src/cli/environmentCommands.ts | 8 +-- .../environmentExportMemory.test.ts | 57 +++++++++++++++++++ 8 files changed, 144 insertions(+), 42 deletions(-) create mode 100644 src/application/environment/environmentEvidenceBytes.ts create mode 100644 tests/process-global/environmentExportMemory.test.ts diff --git a/src/application/environment/environmentBuild.test.ts b/src/application/environment/environmentBuild.test.ts index 0083b158ce..235dce0d3f 100644 --- a/src/application/environment/environmentBuild.test.ts +++ b/src/application/environment/environmentBuild.test.ts @@ -1,3 +1,4 @@ +import { evidenceFileBytes } from "./environmentEvidenceBytes.js"; import { chmod, mkdir, @@ -34,7 +35,6 @@ import { buildEnvironment, canonicalTemplateRules, classifyBuildCapture, - evidenceFileBytes, type EnvironmentBuildInput, } from "./environmentBuild.js"; import { diff --git a/src/application/environment/environmentBuild.ts b/src/application/environment/environmentBuild.ts index dd2e391393..5fd773c115 100644 --- a/src/application/environment/environmentBuild.ts +++ b/src/application/environment/environmentBuild.ts @@ -14,7 +14,6 @@ import { type TemplateRule, } from "../../domain/analysisView/environmentManifestRules.js"; import type { Evidence } from "../../domain/evidence.js"; -import { jsonByteLength, jsonParts } from "../../domain/jsonSerialization.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js"; @@ -32,7 +31,7 @@ import { type EnvironmentBuildWarning, } from "./environmentBuildReport.js"; import { - environmentExportFile, + environmentExportEvidenceFile, exportEnvironmentBundle, packageTarballBytes, type EnvironmentExportFile, @@ -84,18 +83,6 @@ export const classifyBuildCapture = ( ); }; -/** An Evidence file's bytes as `--format json` writes the Evidence to a file: compact JSON and a newline. */ -export const evidenceFileBytes = (evidence: Evidence): Buffer => { - // Measured first, so the document is written into one exactly sized buffer. - const bytes = Buffer.allocUnsafe(jsonByteLength(evidence) + 1); - let offset = 0; - for (const part of jsonParts(evidence)) offset += bytes.write(part, offset); - if (offset !== bytes.length - 1) - throw new Error("Evidence serialized to a different length than measured"); - bytes[offset] = 0x0a; - return bytes; -}; - /** Rules in the views' fixed order, so the report and manifest list them canonically. */ export const canonicalTemplateRules = ( rules: readonly TemplateRule[], @@ -315,10 +302,10 @@ export const buildEnvironment = async ( if (!projected.ok) return projected; const stopped = await cancelled(input.signal); if (stopped !== undefined) return err(stopped); - const manifest = await environmentExportFile( + const manifest = await environmentExportEvidenceFile( "manifest", "manifest.json", - evidenceFileBytes(projected.value), + projected.value, ENVIRONMENT_BUILD_OPERATION, ); if (!manifest.ok) return manifest; diff --git a/src/application/environment/environmentBuildReport.test.ts b/src/application/environment/environmentBuildReport.test.ts index 0fae8dda4a..003f2dfde8 100644 --- a/src/application/environment/environmentBuildReport.test.ts +++ b/src/application/environment/environmentBuildReport.test.ts @@ -196,25 +196,29 @@ describe("scanCredentialNames", () => { }); describe("scanCredentialNames over structured and unattributed records", () => { - it("scans each structured form parameter a HAR records without text", () => { - const structured = record(4, undefined, undefined); - ( - structured.reported as { request: { postData?: unknown } } - ).request.postData = { - mimeType: "multipart/form-data", - params: [ - { name: "api_key", value: "secret" }, - { name: "api_key", value: "" }, - { name: "user", value: "a" }, - { name: "token", value: "t" }, - ], - }; - expect(scanCredentialNames([structured])).toEqual({ - count: 2, - pointers: ["record 4 request /api_key", "record 4 request /token"], - skipped: NONE_SKIPPED, - }); - }); + it.each([undefined, null])( + "scans each structured form parameter with absent text: %s", + (text) => { + const structured = record(4, undefined, undefined); + ( + structured.reported as { request: { postData?: unknown } } + ).request.postData = { + mimeType: "multipart/form-data", + text, + params: [ + { name: "api_key", value: "secret" }, + { name: "api_key", value: "" }, + { name: "user", value: "a" }, + { name: "token", value: "t" }, + ], + }; + expect(scanCredentialNames([structured])).toEqual({ + count: 2, + pointers: ["record 4 request /api_key", "record 4 request /token"], + skipped: NONE_SKIPPED, + }); + }, + ); it("scans the request side of a scenario transaction with duplicate responses", () => { const capture = scenarioCaptureFixture( @@ -258,6 +262,17 @@ describe("scanCredentialNames over structured and unattributed records", () => { }); describe("scanCredentialNames coverage", () => { + it("measures the scan limit in UTF-8 bytes for multibyte text", () => { + const multibyte = record( + 9, + { token: "界".repeat(2 * 1024 * 1024) }, + undefined, + ); + expect(scanCredentialNames([multibyte])).toMatchObject({ + count: 0, + skipped: { count: 1, pointers: ["record 9 request"] }, + }); + }); it("reports text bodies too large to scan by record and side, never by content", () => { const large = JSON.stringify({ token: "t-large", diff --git a/src/application/environment/environmentBuildReport.ts b/src/application/environment/environmentBuildReport.ts index e7460ef509..91f4c2e5a1 100644 --- a/src/application/environment/environmentBuildReport.ts +++ b/src/application/environment/environmentBuildReport.ts @@ -96,7 +96,8 @@ type ScannedBody = /** A recorded body as JSON or form fields, skipped when it is cut or over the limit, or undefined when it is neither. */ const parsedBody = (text: unknown, cut = false): ScannedBody | undefined => { if (typeof text !== "string") return undefined; - if (cut || text.length > SCAN_BODY_LIMIT_BYTES) return { kind: "skipped" }; + if (cut || Buffer.byteLength(text, "utf8") > SCAN_BODY_LIMIT_BYTES) + return { kind: "skipped" }; const start = text.trimStart()[0]; if (start === "{" || start === "[") { try { @@ -203,7 +204,7 @@ export const scanCredentialNames = ( const text = member(postData, "text"); // A HAR may record form fields only as structured params; each occurrence counts, as in a form body. scanBody( - text === undefined + typeof text !== "string" ? { kind: "form", fields: namedValues(member(postData, "params")).flatMap( diff --git a/src/application/environment/environmentEvidenceBytes.ts b/src/application/environment/environmentEvidenceBytes.ts new file mode 100644 index 0000000000..1a4b5654b9 --- /dev/null +++ b/src/application/environment/environmentEvidenceBytes.ts @@ -0,0 +1,19 @@ +import type { Evidence } from "../../domain/evidence.js"; +import { jsonByteLength, jsonParts } from "../../domain/jsonSerialization.js"; + +/** An Evidence file's bytes as `--format json` writes the Evidence to a file: compact JSON and a newline. */ +export const evidenceFileBytes = ( + evidence: Evidence, + admit?: (bytes: number) => void, +): Buffer => { + // Measured first, so the document is written into one exactly sized buffer. + const size = jsonByteLength(evidence) + 1; + admit?.(size); + const bytes = Buffer.allocUnsafe(size); + let offset = 0; + for (const part of jsonParts(evidence)) offset += bytes.write(part, offset); + if (offset !== bytes.length - 1) + throw new Error("Evidence serialized to a different length than measured"); + bytes[offset] = 0x0a; + return bytes; +}; diff --git a/src/application/environment/environmentExport.ts b/src/application/environment/environmentExport.ts index f08d55b730..d3213c7c85 100644 --- a/src/application/environment/environmentExport.ts +++ b/src/application/environment/environmentExport.ts @@ -12,6 +12,8 @@ import { type AnalysisInputIssue, } from "../../domain/analysisErrorCore.js"; import type { JsonValue } from "../../domain/jsonValue.js"; +import type { Evidence } from "../../domain/evidence.js"; +import { evidenceFileBytes } from "./environmentEvidenceBytes.js"; import { err, ok, type Result } from "../../domain/result.js"; import { RegularFileChangedError } from "../../filesystem/RegularFile.js"; import { @@ -227,6 +229,25 @@ const headroomRefusal = ( ); }; +/** Measure and admit every serialized copy before allocating its buffer. */ +export const environmentExportEvidenceFile = async ( + role: "manifest" | "evidence", + path: string, + evidence: Evidence, + operation: string = ENVIRONMENT_EXPORT_OPERATION, +): Promise> => { + try { + const bytes = evidenceFileBytes(evidence, (size) => { + const refused = headroomRefusal(operation, path, size); + if (refused !== undefined) throw refused; + }); + return environmentExportFile(role, path, bytes, operation); + } catch (cause: unknown) { + if (cause instanceof AnalysisResourceConstraintError) return err(cause); + throw cause; + } +}; + /** Parse and seal bytes the bundle will copy, so authentication reads exactly what ships. */ export const environmentExportFile = async ( role: "manifest" | "evidence", @@ -234,6 +255,8 @@ export const environmentExportFile = async ( bytes: Buffer, operation: string = ENVIRONMENT_EXPORT_OPERATION, ): Promise> => { + const refusal = headroomRefusal(operation, path, bytes.length); + if (refusal !== undefined) return err(refusal); let parsed: ReturnType; try { parsed = parseUtf8Json(bytes, operation, path); diff --git a/src/cli/environmentCommands.ts b/src/cli/environmentCommands.ts index cf26e81bcd..6445523de5 100644 --- a/src/cli/environmentCommands.ts +++ b/src/cli/environmentCommands.ts @@ -354,9 +354,9 @@ const inspectBuildHar = async ( const { createWebNetworkCaptureService } = await import("../composition/webNetworkCaptures.js"); const { parseEvidence } = await import("../domain/evidence.js"); - const { environmentExportFile } = + const { environmentExportEvidenceFile } = await import("../application/environment/environmentExport.js"); - const { evidenceFileBytes, ENVIRONMENT_BUILD_OPERATION } = + const { ENVIRONMENT_BUILD_OPERATION } = await import("../application/environment/environmentBuild.js"); const service = createWebNetworkCaptureService(environment); const inspected = await finishAnalysisCommand( @@ -381,10 +381,10 @@ const inspectBuildHar = async ( error: "Application workflow failed", ...(inspected as Record), }); - const file = await environmentExportFile( + const file = await environmentExportEvidenceFile( "evidence", capturePath, - evidenceFileBytes(parseEvidence(inspected)), + parseEvidence(inspected), ENVIRONMENT_BUILD_OPERATION, ); return file.ok ? ok(file.value) : err(cliError(file.error)); diff --git a/tests/process-global/environmentExportMemory.test.ts b/tests/process-global/environmentExportMemory.test.ts new file mode 100644 index 0000000000..cae4f99a05 --- /dev/null +++ b/tests/process-global/environmentExportMemory.test.ts @@ -0,0 +1,57 @@ +import { getHeapStatistics } from "node:v8"; + +import { expect, it, vi } from "vitest"; + +import { evidenceFileBytes } from "../../src/application/environment/environmentEvidenceBytes.js"; +import { + environmentExportEvidenceFile, + environmentExportFile, +} from "../../src/application/environment/environmentExport.js"; +import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; +import { + analysisViewCaptureEvidence, + environmentManifestCaptureFixture, +} from "../fixtures/analysisView.js"; + +vi.mock("node:v8", { spy: true }); + +it.each(["serialize", "parse"] as const)( + "refuses in-memory export before %s when its copies cannot fit", + async (stage) => { + const capture = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(), + ); + const bytes = evidenceFileBytes(capture); + const statistics = getHeapStatistics(); + const allocate = vi.spyOn(Buffer, "allocUnsafe"); + vi.mocked(getHeapStatistics).mockReturnValue({ + ...statistics, + total_available_size: 1, + }); + try { + const result = + stage === "serialize" + ? await environmentExportEvidenceFile( + "evidence", + "capture.json", + capture, + "environment-build", + ) + : await environmentExportFile( + "evidence", + "capture.json", + bytes, + "environment-build", + ); + if (result.ok) throw new Error("Expected a memory refusal"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "resource_constraint", + details: { operation: "environment-build", resource: "memory" }, + }); + expect(allocate).not.toHaveBeenCalled(); + } finally { + vi.mocked(getHeapStatistics).mockReset(); + allocate.mockRestore(); + } + }, +);