From fdc4e3908b70d7cf6b03f357fb9c42a133bb5577 Mon Sep 17 00:00:00 2001 From: Sahinur <68515168+SahinurDEV@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:12:31 +0600 Subject: [PATCH] fix(artifacts): report ZIP Zstd window refusals as resource limits Classify a native method-93 decoder window refusal once at the ZIP owner as limit/memory, retaining the fixed 128 MiB cap, native cause and existing corruption/CRC/size/cancellation handling. Derive recovery labels from the cap and share one independent frame fixture. Validated with 43 reader/translationtests,three actual CLI/stdio MCP cases, a pinned public GLFW payload rewrapped in independently verified ZIP controls, six equivalent CLI calls before/after, static checks and documentation checks. All input bytes remain unchanged, normal extraction matches exactly, and failed outputs are absent. ELF/GPU component refusal-policy gaps remain separate unverified full-format workflows. The original three unchecked human declarations are preserved. The explicitly authorized readiness-only exception applies after technical CI and review; no human-review attestation is added. Fixes #8623 Full technical CI 42279406899 passes, including all eight shards and required aggregates. Greptile reviews this head at 5/5; Codex cannot complete because the repository review quota is exhausted. The Author readiness-only exception is explicitly authorized; no human-review attestation is added. Assisted-by: Reimu (gensokyo-skills:reimu-incident-triage) --- docs/cli.md | 5 +- src/artifacts/ZipArtifactReader.test.ts | 112 +++++++++++----- src/artifacts/ZipArtifactReader.ts | 21 ++- .../cli/artifactZstandardWindow.test.ts | 120 ++++++++++++++++++ tests/fixtures/zipZstandard.ts | 11 ++ 5 files changed, 235 insertions(+), 34 deletions(-) create mode 100644 tests/boundary/cli/artifactZstandardWindow.test.ts diff --git a/docs/cli.md b/docs/cli.md index e0b8d0e106..6e5c556d1c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -349,7 +349,10 @@ unknown names, resource limits and retained views. Compressed bytes stay stored. Zstandard (standard method 93). They retain the shared ZIP CRC, declared-size, overlap and archive expansion checks. The native streaming decoder keeps the existing 128 MiB Zstandard window cap; frames requiring a larger window are -refused. Deprecated method 20 and other unregistered ZIP codecs remain unsupported. +reported as a resource limit (`reason: limit`, `boundary: memory`), rather than +as archive corruption. The cap cannot be raised at runtime: recompress with a +window of at most 128 MiB, or decompress externally before retrying. +Deprecated method 20 and other unregistered ZIP codecs remain unsupported. Inventory and extract UF2 or DfuSe regions: diff --git a/src/artifacts/ZipArtifactReader.test.ts b/src/artifacts/ZipArtifactReader.test.ts index a117ea9ba3..f629a0a390 100644 --- a/src/artifacts/ZipArtifactReader.test.ts +++ b/src/artifacts/ZipArtifactReader.test.ts @@ -1,6 +1,9 @@ import { writeOrderedZip } from "../../tests/fixtures/artifactEntryOrder.js"; import { zipSymlinkMetadataFixture } from "../../tests/fixtures/zipSymlinkMetadata.js"; -import { zstandardZipFixture } from "../../tests/fixtures/zipZstandard.js"; +import { + zstandardZipFixture, + zstandardWindowFrameFixture, +} from "../../tests/fixtures/zipZstandard.js"; import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js"; import { createDeferred } from "../../tests/fixtures/binarySession.js"; import { once } from "node:events"; @@ -27,6 +30,7 @@ import { MINIMUM_ARTIFACT_EXPANDED_BYTES } from "../domain/artifactExpansionBudg import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { hashReadable } from "./ArtifactHash.js"; import { translateArtifactFailure } from "./ArtifactOperationFailure.js"; +import { analysisErrorUserMessage } from "../domain/analysisErrorPresentation.js"; /** Small real ZIP carriers with forged sizes; metadata-only controls, never decompressed. */ const writeZipWithDeclaredSize = async ( @@ -115,35 +119,38 @@ const admittedZip = async () => { return { path, content, handle, positional, initial }; }; -describe("ZIP Zstandard method 93", () => { - const read = async (archive: Buffer): Promise => { - const root = await createTestTempDirectory("rea-zip-zstd-"); - const path = join(root, "method93.zip"); - await writeFile(path, archive); - const reader = new ZipArtifactReader(path, "zip"); - try { - const entries = []; - for await (const entry of reader.entries()) entries.push(entry); - const entry = entries[0]; - if (entry === undefined) throw new Error("Missing ZIP member"); - const chunks: Buffer[] = []; - for await (const chunk of await reader.open(entry)) - chunks.push(Buffer.from(chunk)); - return Buffer.concat(chunks); - } finally { - await reader.close(); - } - }; +/** Read the first member of one method-93 ZIP fixture. */ +const readFirstZstandardMember = async (archive: Buffer): Promise => { + const root = await createTestTempDirectory("rea-zip-zstd-"); + const path = join(root, "method93.zip"); + await writeFile(path, archive); + const reader = new ZipArtifactReader(path, "zip"); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const entry = entries[0]; + if (entry === undefined) throw new Error("Missing ZIP member"); + const chunks: Buffer[] = []; + for await (const chunk of await reader.open(entry)) + chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); + } finally { + await reader.close(); + } +}; +describe("ZIP Zstandard method 93", () => { it.each([ Buffer.alloc(0), Buffer.from("Zstandard ZIP evidence: café\n".repeat(7)), ])( "streams exact bytes from a standard method-93 ZIP: %j", async (content) => { - expect((await read(zstandardZipFixture(content))).equals(content)).toBe( - true, - ); + expect( + (await readFirstZstandardMember(zstandardZipFixture(content))).equals( + content, + ), + ).toBe(true); }, ); @@ -154,7 +161,9 @@ describe("ZIP Zstandard method 93", () => { "retains %s validation for Zstandard entries", async (_label, options, message) => { await expect( - read(zstandardZipFixture(Buffer.from("member bytes"), options)), + readFirstZstandardMember( + zstandardZipFixture(Buffer.from("member bytes"), options), + ), ).rejects.toMatchObject({ name: "ArtifactReaderFailure", reason: "format", @@ -167,7 +176,9 @@ describe("ZIP Zstandard method 93", () => { "keeps unsupported ZIP method %s unavailable", async (method) => { await expect( - read(zstandardZipFixture(Buffer.from("member bytes"), { method })), + readFirstZstandardMember( + zstandardZipFixture(Buffer.from("member bytes"), { method }), + ), ).rejects.toMatchObject({ reason: "unavailable", message: expect.stringContaining("Compression method not supported"), @@ -178,10 +189,11 @@ describe("ZIP Zstandard method 93", () => { it.each([ ["bad frame", Buffer.from("not a Zstandard frame")], ["truncated frame", Buffer.from("28b52ffd", "hex")], - ["excessive decoder window", Buffer.from("28b52ffd0090010000", "hex")], ] as const)("reports %s as an archive fault", async (_label, compressed) => { await expect( - read(zstandardZipFixture(Buffer.alloc(0), { compressed })), + readFirstZstandardMember( + zstandardZipFixture(Buffer.alloc(0), { compressed }), + ), ).rejects.toMatchObject({ name: "ArtifactReaderFailure", reason: "format", @@ -196,7 +208,7 @@ describe("ZIP Zstandard method 93", () => { }); compressed[compressed.length - 1]! ^= 1; await expect( - read(zstandardZipFixture(content, { compressed })), + readFirstZstandardMember(zstandardZipFixture(content, { compressed })), ).rejects.toMatchObject({ reason: "format", message: expect.stringContaining("Invalid compressed data"), @@ -206,7 +218,7 @@ describe("ZIP Zstandard method 93", () => { it("reports a required Zstandard dictionary as unavailable", async () => { const compressed = Buffer.from("28b52ffd210100010000", "hex"); - const failure = await read( + const failure = await readFirstZstandardMember( zstandardZipFixture(Buffer.alloc(0), { compressed }), ).catch((cause: unknown) => cause); expect(failure).toMatchObject({ @@ -243,7 +255,7 @@ describe("ZIP Zstandard method 93", () => { "member bytes member bytes member bytes member bytes", ); await expect( - read( + readFirstZstandardMember( zstandardZipFixture(content, { compressed: Buffer.from(hex, "hex") }), ), ).rejects.toMatchObject({ @@ -256,7 +268,7 @@ describe("ZIP Zstandard method 93", () => { it("retains the archive expansion budget for method-93 declarations", async () => { await expect( - read( + readFirstZstandardMember( zstandardZipFixture(Buffer.from("small"), { declaredSize: MINIMUM_ARTIFACT_EXPANDED_BYTES + 1, }), @@ -267,6 +279,44 @@ describe("ZIP Zstandard method 93", () => { }); }); +describe("ZIP Zstandard decoder window", () => { + it("reports a valid frame above the decoder window as a resource limit", async () => { + // Window descriptor 0x90 declares 256 MiB; the single raw block is valid. + const { content, compressed: largeWindow } = + zstandardWindowFrameFixture(0x90); + const failure = await readFirstZstandardMember( + zstandardZipFixture(content, { compressed: largeWindow }), + ).catch((cause: unknown) => cause); + expect(failure).toMatchObject({ + name: "ArtifactReaderFailure", + reason: "limit", + boundary: "memory", + message: expect.stringContaining("fixed 128 MiB allowance"), + cause: { code: "ZSTD_error_frameParameter_windowTooLarge" }, + }); + const translated = translateArtifactFailure("extract_artifact", failure); + expect(translated).toMatchObject({ + reason: "limit", + boundary: "memory", + detail: expect.stringContaining("fixed 128 MiB allowance"), + }); + const userMessage = analysisErrorUserMessage(translated); + expect(userMessage).toContain("fixed 128 MiB allowance"); + expect(userMessage).not.toContain("fresh copy"); + + // Control: the same frame with a 1 MiB window descriptor decodes exactly. + const smallWindow = Buffer.from(largeWindow); + smallWindow[5] = 0x50; + expect( + ( + await readFirstZstandardMember( + zstandardZipFixture(content, { compressed: smallWindow }), + ) + ).equals(content), + ).toBe(true); + }); +}); + describe("ZIP Zstandard member streams", () => { it.each(["abort", "consumer-close"] as const)( "settles a streaming Zstandard read after %s and allows another read", diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index e1b4e2c2ca..e8d0a61137 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -53,6 +53,14 @@ const ZIP_UTF8_DECODER = new TextDecoder("utf-8", { ignoreBOM: true, }); +/** Fixed method-93 decoder window: 2^27 bytes, matching the ELF/GPU decoders. */ +const ZIP_ZSTANDARD_WINDOW_LOG_MAX = 27; + +const ZIP_ZSTANDARD_WINDOW_ALLOWANCE = `${String(2 ** (ZIP_ZSTANDARD_WINDOW_LOG_MAX - 20))} MiB`; + +/** A frame above the window cap is a deliberate resource refusal, not damage. */ +const ZIP_ZSTANDARD_WINDOW_LIMIT = `ZIP Zstandard member requires a decoder window above REA's fixed ${ZIP_ZSTANDARD_WINDOW_ALLOWANCE} allowance; the archive is not necessarily corrupt, and this limit cannot be raised at runtime. Recompress the member with a window of at most ${ZIP_ZSTANDARD_WINDOW_ALLOWANCE}, or decompress it with a tool that admits larger windows, then retry.`; + /** Keep zip.js's framing, CRC, size checks and backpressure for method 93. */ class ZipZstandardDecompressionStream { readonly readable; @@ -63,7 +71,9 @@ class ZipZstandardDecompressionStream { createZstdDecompress({ chunkSize: options?.chunkSize, // Match the existing ELF/GPU Zstandard decoder's 128 MiB window cap. - params: { [constants.ZSTD_d_windowLogMax]: 27 }, + params: { + [constants.ZSTD_d_windowLogMax]: ZIP_ZSTANDARD_WINDOW_LOG_MAX, + }, }), ); this.readable = stream.readable; @@ -525,6 +535,14 @@ const typedZipFailure = ( "ZIP archive uses an unsupported Zstandard dictionary", { cause }, ); + if ( + compressionMethod === 93 && + code === "ZSTD_error_frameParameter_windowTooLarge" + ) + return new ArtifactReaderFailure("limit", ZIP_ZSTANDARD_WINDOW_LIMIT, { + cause, + boundary: "memory", + }); const invalidZstandard = compressionMethod === 93 && (code === "ZSTD_error_prefix_unknown" || @@ -532,7 +550,6 @@ const typedZipFailure = ( code === "ZSTD_error_corruption_detected" || code === "ZSTD_error_dictionary_corrupted" || code === "ZSTD_error_frameParameter_unsupported" || - code === "ZSTD_error_frameParameter_windowTooLarge" || code === "Z_BUF_ERROR"); const reason = ZIP_FAILURE_REASONS.get(cause.message) ?? diff --git a/tests/boundary/cli/artifactZstandardWindow.test.ts b/tests/boundary/cli/artifactZstandardWindow.test.ts new file mode 100644 index 0000000000..ed2433c0d9 --- /dev/null +++ b/tests/boundary/cli/artifactZstandardWindow.test.ts @@ -0,0 +1,120 @@ +import { createHash } from "node:crypto"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { parseEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + zstandardWindowFrameFixture, + zstandardZipFixture, +} from "../../fixtures/zipZstandard.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const expectRefusal = (diagnostic: unknown, large: boolean): void => { + expect(diagnostic).toMatchObject({ + code: large ? "truncated" : "artifact_operation_failed", + details: { + reason: large ? "limit" : "format", + ...(large ? { boundary: "memory", truncated: true } : {}), + }, + }); + if (large) { + expect(diagnostic).toMatchObject({ + message: expect.stringContaining("fixed 128 MiB allowance"), + remediation: { + action: expect.stringContaining("cannot be raised at runtime"), + }, + }); + expect(JSON.stringify(diagnostic)).not.toContain("fresh copy"); + } +}; + +cliTest.for(["small", "large", "corrupt"] as const)( + "preserves ZIP window classification over CLI and stdio MCP: %s", + async (kind, { cli }) => { + const root = await createTestTempDirectory("rea-zstd-window-"); + const path = join(root, "fixture.zip"); + const { content, compressed } = zstandardWindowFrameFixture( + kind === "large" ? 0x90 : 0x50, + ); + if (kind === "corrupt") compressed[0] = 0; + const bytes = zstandardZipFixture(content, { compressed }); + await writeFile(path, bytes); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const success = kind === "small"; + const evidencePath = join(root, "inventory.json"); + for (const operation of ["inspect-artifact", "extract-artifact"] as const) { + const outputRoot = join(root, `cli-${operation}`); + const reply = await cli.run({ + arguments: [ + operation, + path, + "--json", + ...(operation === "inspect-artifact" + ? ["--detail", "summary", "--evidence_output", evidencePath] + : ["--output-root", outputRoot]), + ], + }); + expect(reply.exitCode, reply.stderr).toBe(success ? 0 : 1); + if (!success) expectRefusal(reply.json, kind === "large"); + else if (operation === "extract-artifact") + expect( + (await readFile(join(outputRoot, "member.txt"))).equals(content), + ).toBe(true); + else { + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(evidencePath, "utf8")), + ); + expect(bundle.records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + subject: expect.objectContaining({ + digest: expect.objectContaining({ sha256 }), + }), + }), + ]), + ); + } + if (!success && operation === "extract-artifact") + await expect(access(outputRoot)).rejects.toMatchObject({ + code: "ENOENT", + }); + } + + const call = await cli.sharedMcpCall!(); + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + try { + for (const operation of [ + "inspect_artifact", + "extract_artifact", + ] as const) { + const outputRoot = join(root, `mcp-${operation}`); + const reply = await call( + operation, + operation === "inspect_artifact" + ? { detail: "summary" } + : { output_root: outputRoot }, + ); + if (!success) { + expectRefusal(parseMcpToolError(reply).error, kind === "large"); + if (operation === "extract_artifact") + await expect(access(outputRoot)).rejects.toMatchObject({ + code: "ENOENT", + }); + } else { + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + if (operation === "extract_artifact") + expect( + (await readFile(join(outputRoot, "member.txt"))).equals(content), + ).toBe(true); + } + } + } finally { + expect((await call("close_binary", {})).isError).not.toBe(true); + } + expect((await readFile(path)).equals(bytes)).toBe(true); + }, +); diff --git a/tests/fixtures/zipZstandard.ts b/tests/fixtures/zipZstandard.ts index 4499586616..ee7c149871 100644 --- a/tests/fixtures/zipZstandard.ts +++ b/tests/fixtures/zipZstandard.ts @@ -1,5 +1,16 @@ import { crc32, zstdCompressSync } from "node:zlib"; +/** A valid independent raw-block frame; its window is unrelated to its tiny output. */ +export const zstandardWindowFrameFixture = (windowDescriptor: number) => { + const content = Buffer.from("REA independent frame control\n"); + const compressed = Buffer.from( + "28b52ffd0090f1000052454120696e646570656e64656e74206672616d6520636f6e74726f6c0a", + "hex", + ); + compressed[5] = windowDescriptor; + return { content, compressed }; +}; + /** One non-Zip64 member, independently framed from the PKWARE APPNOTE records. */ export const zstandardZipFixture = ( content: Buffer,