mirror of
https://github.com/morluto/rea.git
synced 2026-11-06 23:06:18 +00:00
fix(artifacts): report ZIP Zstd window refusals as resource limits
Classify a native method-93 decoder window refusal once at the ZIP owner as limit/memory, retaining the fixed 128 MiB cap, native cause and existing corruption/CRC/size/cancellation handling. Derive recovery labels from the cap and share one independent frame fixture. Validated with 43 reader/translationtests,three actual CLI/stdio MCP cases, a pinned public GLFW payload rewrapped in independently verified ZIP controls, six equivalent CLI calls before/after, static checks and documentation checks. All input bytes remain unchanged, normal extraction matches exactly, and failed outputs are absent. ELF/GPU component refusal-policy gaps remain separate unverified full-format workflows. The original three unchecked human declarations are preserved. The explicitly authorized readiness-only exception applies after technical CI and review; no human-review attestation is added. Fixes #8623 Full technical CI 42279406899 passes, including all eight shards and required aggregates. Greptile reviews this head at 5/5; Codex cannot complete because the repository review quota is exhausted. The Author readiness-only exception is explicitly authorized; no human-review attestation is added. Assisted-by: Reimu (gensokyo-skills:reimu-incident-triage)
This commit is contained in:
1 parent
81ada025c0
commit
fdc4e3908b
5 files changed
+235
-34
No files matched your search
+4
-1
@@ -349,7 +349,10 @@ unknown names, resource limits and retained views. Compressed bytes stay stored.
|
||||
Zstandard (standard method 93). They retain the shared ZIP CRC, declared-size,
|
||||
overlap and archive expansion checks. The native streaming decoder keeps the
|
||||
existing 128 MiB Zstandard window cap; frames requiring a larger window are
|
||||
refused. Deprecated method 20 and other unregistered ZIP codecs remain unsupported.
|
||||
reported as a resource limit (`reason: limit`, `boundary: memory`), rather than
|
||||
as archive corruption. The cap cannot be raised at runtime: recompress with a
|
||||
window of at most 128 MiB, or decompress externally before retrying.
|
||||
Deprecated method 20 and other unregistered ZIP codecs remain unsupported.
|
||||
|
||||
Inventory and extract UF2 or DfuSe regions:
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { writeOrderedZip } from "../../tests/fixtures/artifactEntryOrder.js";
|
||||
import { zipSymlinkMetadataFixture } from "../../tests/fixtures/zipSymlinkMetadata.js";
|
||||
import { zstandardZipFixture } from "../../tests/fixtures/zipZstandard.js";
|
||||
import {
|
||||
zstandardZipFixture,
|
||||
zstandardWindowFrameFixture,
|
||||
} from "../../tests/fixtures/zipZstandard.js";
|
||||
import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js";
|
||||
import { createDeferred } from "../../tests/fixtures/binarySession.js";
|
||||
import { once } from "node:events";
|
||||
@@ -27,6 +30,7 @@ import { MINIMUM_ARTIFACT_EXPANDED_BYTES } from "../domain/artifactExpansionBudg
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { hashReadable } from "./ArtifactHash.js";
|
||||
import { translateArtifactFailure } from "./ArtifactOperationFailure.js";
|
||||
import { analysisErrorUserMessage } from "../domain/analysisErrorPresentation.js";
|
||||
|
||||
/** Small real ZIP carriers with forged sizes; metadata-only controls, never decompressed. */
|
||||
const writeZipWithDeclaredSize = async (
|
||||
@@ -115,35 +119,38 @@ const admittedZip = async () => {
|
||||
return { path, content, handle, positional, initial };
|
||||
};
|
||||
|
||||
describe("ZIP Zstandard method 93", () => {
|
||||
const read = async (archive: Buffer): Promise<Buffer> => {
|
||||
const root = await createTestTempDirectory("rea-zip-zstd-");
|
||||
const path = join(root, "method93.zip");
|
||||
await writeFile(path, archive);
|
||||
const reader = new ZipArtifactReader(path, "zip");
|
||||
try {
|
||||
const entries = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const entry = entries[0];
|
||||
if (entry === undefined) throw new Error("Missing ZIP member");
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of await reader.open(entry))
|
||||
chunks.push(Buffer.from(chunk));
|
||||
return Buffer.concat(chunks);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
};
|
||||
/** Read the first member of one method-93 ZIP fixture. */
|
||||
const readFirstZstandardMember = async (archive: Buffer): Promise<Buffer> => {
|
||||
const root = await createTestTempDirectory("rea-zip-zstd-");
|
||||
const path = join(root, "method93.zip");
|
||||
await writeFile(path, archive);
|
||||
const reader = new ZipArtifactReader(path, "zip");
|
||||
try {
|
||||
const entries = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const entry = entries[0];
|
||||
if (entry === undefined) throw new Error("Missing ZIP member");
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of await reader.open(entry))
|
||||
chunks.push(Buffer.from(chunk));
|
||||
return Buffer.concat(chunks);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
};
|
||||
|
||||
describe("ZIP Zstandard method 93", () => {
|
||||
it.each([
|
||||
Buffer.alloc(0),
|
||||
Buffer.from("Zstandard ZIP evidence: café\n".repeat(7)),
|
||||
])(
|
||||
"streams exact bytes from a standard method-93 ZIP: %j",
|
||||
async (content) => {
|
||||
expect((await read(zstandardZipFixture(content))).equals(content)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
(await readFirstZstandardMember(zstandardZipFixture(content))).equals(
|
||||
content,
|
||||
),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -154,7 +161,9 @@ describe("ZIP Zstandard method 93", () => {
|
||||
"retains %s validation for Zstandard entries",
|
||||
async (_label, options, message) => {
|
||||
await expect(
|
||||
read(zstandardZipFixture(Buffer.from("member bytes"), options)),
|
||||
readFirstZstandardMember(
|
||||
zstandardZipFixture(Buffer.from("member bytes"), options),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
name: "ArtifactReaderFailure",
|
||||
reason: "format",
|
||||
@@ -167,7 +176,9 @@ describe("ZIP Zstandard method 93", () => {
|
||||
"keeps unsupported ZIP method %s unavailable",
|
||||
async (method) => {
|
||||
await expect(
|
||||
read(zstandardZipFixture(Buffer.from("member bytes"), { method })),
|
||||
readFirstZstandardMember(
|
||||
zstandardZipFixture(Buffer.from("member bytes"), { method }),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
message: expect.stringContaining("Compression method not supported"),
|
||||
@@ -178,10 +189,11 @@ describe("ZIP Zstandard method 93", () => {
|
||||
it.each([
|
||||
["bad frame", Buffer.from("not a Zstandard frame")],
|
||||
["truncated frame", Buffer.from("28b52ffd", "hex")],
|
||||
["excessive decoder window", Buffer.from("28b52ffd0090010000", "hex")],
|
||||
] as const)("reports %s as an archive fault", async (_label, compressed) => {
|
||||
await expect(
|
||||
read(zstandardZipFixture(Buffer.alloc(0), { compressed })),
|
||||
readFirstZstandardMember(
|
||||
zstandardZipFixture(Buffer.alloc(0), { compressed }),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
name: "ArtifactReaderFailure",
|
||||
reason: "format",
|
||||
@@ -196,7 +208,7 @@ describe("ZIP Zstandard method 93", () => {
|
||||
});
|
||||
compressed[compressed.length - 1]! ^= 1;
|
||||
await expect(
|
||||
read(zstandardZipFixture(content, { compressed })),
|
||||
readFirstZstandardMember(zstandardZipFixture(content, { compressed })),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("Invalid compressed data"),
|
||||
@@ -206,7 +218,7 @@ describe("ZIP Zstandard method 93", () => {
|
||||
|
||||
it("reports a required Zstandard dictionary as unavailable", async () => {
|
||||
const compressed = Buffer.from("28b52ffd210100010000", "hex");
|
||||
const failure = await read(
|
||||
const failure = await readFirstZstandardMember(
|
||||
zstandardZipFixture(Buffer.alloc(0), { compressed }),
|
||||
).catch((cause: unknown) => cause);
|
||||
expect(failure).toMatchObject({
|
||||
@@ -243,7 +255,7 @@ describe("ZIP Zstandard method 93", () => {
|
||||
"member bytes member bytes member bytes member bytes",
|
||||
);
|
||||
await expect(
|
||||
read(
|
||||
readFirstZstandardMember(
|
||||
zstandardZipFixture(content, { compressed: Buffer.from(hex, "hex") }),
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
@@ -256,7 +268,7 @@ describe("ZIP Zstandard method 93", () => {
|
||||
|
||||
it("retains the archive expansion budget for method-93 declarations", async () => {
|
||||
await expect(
|
||||
read(
|
||||
readFirstZstandardMember(
|
||||
zstandardZipFixture(Buffer.from("small"), {
|
||||
declaredSize: MINIMUM_ARTIFACT_EXPANDED_BYTES + 1,
|
||||
}),
|
||||
@@ -267,6 +279,44 @@ describe("ZIP Zstandard method 93", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("ZIP Zstandard decoder window", () => {
|
||||
it("reports a valid frame above the decoder window as a resource limit", async () => {
|
||||
// Window descriptor 0x90 declares 256 MiB; the single raw block is valid.
|
||||
const { content, compressed: largeWindow } =
|
||||
zstandardWindowFrameFixture(0x90);
|
||||
const failure = await readFirstZstandardMember(
|
||||
zstandardZipFixture(content, { compressed: largeWindow }),
|
||||
).catch((cause: unknown) => cause);
|
||||
expect(failure).toMatchObject({
|
||||
name: "ArtifactReaderFailure",
|
||||
reason: "limit",
|
||||
boundary: "memory",
|
||||
message: expect.stringContaining("fixed 128 MiB allowance"),
|
||||
cause: { code: "ZSTD_error_frameParameter_windowTooLarge" },
|
||||
});
|
||||
const translated = translateArtifactFailure("extract_artifact", failure);
|
||||
expect(translated).toMatchObject({
|
||||
reason: "limit",
|
||||
boundary: "memory",
|
||||
detail: expect.stringContaining("fixed 128 MiB allowance"),
|
||||
});
|
||||
const userMessage = analysisErrorUserMessage(translated);
|
||||
expect(userMessage).toContain("fixed 128 MiB allowance");
|
||||
expect(userMessage).not.toContain("fresh copy");
|
||||
|
||||
// Control: the same frame with a 1 MiB window descriptor decodes exactly.
|
||||
const smallWindow = Buffer.from(largeWindow);
|
||||
smallWindow[5] = 0x50;
|
||||
expect(
|
||||
(
|
||||
await readFirstZstandardMember(
|
||||
zstandardZipFixture(content, { compressed: smallWindow }),
|
||||
)
|
||||
).equals(content),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ZIP Zstandard member streams", () => {
|
||||
it.each(["abort", "consumer-close"] as const)(
|
||||
"settles a streaming Zstandard read after %s and allows another read",
|
||||
|
||||
@@ -53,6 +53,14 @@ const ZIP_UTF8_DECODER = new TextDecoder("utf-8", {
|
||||
ignoreBOM: true,
|
||||
});
|
||||
|
||||
/** Fixed method-93 decoder window: 2^27 bytes, matching the ELF/GPU decoders. */
|
||||
const ZIP_ZSTANDARD_WINDOW_LOG_MAX = 27;
|
||||
|
||||
const ZIP_ZSTANDARD_WINDOW_ALLOWANCE = `${String(2 ** (ZIP_ZSTANDARD_WINDOW_LOG_MAX - 20))} MiB`;
|
||||
|
||||
/** A frame above the window cap is a deliberate resource refusal, not damage. */
|
||||
const ZIP_ZSTANDARD_WINDOW_LIMIT = `ZIP Zstandard member requires a decoder window above REA's fixed ${ZIP_ZSTANDARD_WINDOW_ALLOWANCE} allowance; the archive is not necessarily corrupt, and this limit cannot be raised at runtime. Recompress the member with a window of at most ${ZIP_ZSTANDARD_WINDOW_ALLOWANCE}, or decompress it with a tool that admits larger windows, then retry.`;
|
||||
|
||||
/** Keep zip.js's framing, CRC, size checks and backpressure for method 93. */
|
||||
class ZipZstandardDecompressionStream {
|
||||
readonly readable;
|
||||
@@ -63,7 +71,9 @@ class ZipZstandardDecompressionStream {
|
||||
createZstdDecompress({
|
||||
chunkSize: options?.chunkSize,
|
||||
// Match the existing ELF/GPU Zstandard decoder's 128 MiB window cap.
|
||||
params: { [constants.ZSTD_d_windowLogMax]: 27 },
|
||||
params: {
|
||||
[constants.ZSTD_d_windowLogMax]: ZIP_ZSTANDARD_WINDOW_LOG_MAX,
|
||||
},
|
||||
}),
|
||||
);
|
||||
this.readable = stream.readable;
|
||||
@@ -525,6 +535,14 @@ const typedZipFailure = (
|
||||
"ZIP archive uses an unsupported Zstandard dictionary",
|
||||
{ cause },
|
||||
);
|
||||
if (
|
||||
compressionMethod === 93 &&
|
||||
code === "ZSTD_error_frameParameter_windowTooLarge"
|
||||
)
|
||||
return new ArtifactReaderFailure("limit", ZIP_ZSTANDARD_WINDOW_LIMIT, {
|
||||
cause,
|
||||
boundary: "memory",
|
||||
});
|
||||
const invalidZstandard =
|
||||
compressionMethod === 93 &&
|
||||
(code === "ZSTD_error_prefix_unknown" ||
|
||||
@@ -532,7 +550,6 @@ const typedZipFailure = (
|
||||
code === "ZSTD_error_corruption_detected" ||
|
||||
code === "ZSTD_error_dictionary_corrupted" ||
|
||||
code === "ZSTD_error_frameParameter_unsupported" ||
|
||||
code === "ZSTD_error_frameParameter_windowTooLarge" ||
|
||||
code === "Z_BUF_ERROR");
|
||||
const reason =
|
||||
ZIP_FAILURE_REASONS.get(cause.message) ??
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { access, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect } from "vitest";
|
||||
|
||||
import { parseEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
zstandardWindowFrameFixture,
|
||||
zstandardZipFixture,
|
||||
} from "../../fixtures/zipZstandard.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
|
||||
const expectRefusal = (diagnostic: unknown, large: boolean): void => {
|
||||
expect(diagnostic).toMatchObject({
|
||||
code: large ? "truncated" : "artifact_operation_failed",
|
||||
details: {
|
||||
reason: large ? "limit" : "format",
|
||||
...(large ? { boundary: "memory", truncated: true } : {}),
|
||||
},
|
||||
});
|
||||
if (large) {
|
||||
expect(diagnostic).toMatchObject({
|
||||
message: expect.stringContaining("fixed 128 MiB allowance"),
|
||||
remediation: {
|
||||
action: expect.stringContaining("cannot be raised at runtime"),
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(diagnostic)).not.toContain("fresh copy");
|
||||
}
|
||||
};
|
||||
|
||||
cliTest.for(["small", "large", "corrupt"] as const)(
|
||||
"preserves ZIP window classification over CLI and stdio MCP: %s",
|
||||
async (kind, { cli }) => {
|
||||
const root = await createTestTempDirectory("rea-zstd-window-");
|
||||
const path = join(root, "fixture.zip");
|
||||
const { content, compressed } = zstandardWindowFrameFixture(
|
||||
kind === "large" ? 0x90 : 0x50,
|
||||
);
|
||||
if (kind === "corrupt") compressed[0] = 0;
|
||||
const bytes = zstandardZipFixture(content, { compressed });
|
||||
await writeFile(path, bytes);
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
const success = kind === "small";
|
||||
const evidencePath = join(root, "inventory.json");
|
||||
for (const operation of ["inspect-artifact", "extract-artifact"] as const) {
|
||||
const outputRoot = join(root, `cli-${operation}`);
|
||||
const reply = await cli.run({
|
||||
arguments: [
|
||||
operation,
|
||||
path,
|
||||
"--json",
|
||||
...(operation === "inspect-artifact"
|
||||
? ["--detail", "summary", "--evidence_output", evidencePath]
|
||||
: ["--output-root", outputRoot]),
|
||||
],
|
||||
});
|
||||
expect(reply.exitCode, reply.stderr).toBe(success ? 0 : 1);
|
||||
if (!success) expectRefusal(reply.json, kind === "large");
|
||||
else if (operation === "extract-artifact")
|
||||
expect(
|
||||
(await readFile(join(outputRoot, "member.txt"))).equals(content),
|
||||
).toBe(true);
|
||||
else {
|
||||
const bundle = parseEvidenceBundle(
|
||||
JSON.parse(await readFile(evidencePath, "utf8")),
|
||||
);
|
||||
expect(bundle.records).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
subject: expect.objectContaining({
|
||||
digest: expect.objectContaining({ sha256 }),
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
}
|
||||
if (!success && operation === "extract-artifact")
|
||||
await expect(access(outputRoot)).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
}
|
||||
|
||||
const call = await cli.sharedMcpCall!();
|
||||
const opened = await call("open_binary", { path });
|
||||
expect(opened.isError, JSON.stringify(opened)).not.toBe(true);
|
||||
try {
|
||||
for (const operation of [
|
||||
"inspect_artifact",
|
||||
"extract_artifact",
|
||||
] as const) {
|
||||
const outputRoot = join(root, `mcp-${operation}`);
|
||||
const reply = await call(
|
||||
operation,
|
||||
operation === "inspect_artifact"
|
||||
? { detail: "summary" }
|
||||
: { output_root: outputRoot },
|
||||
);
|
||||
if (!success) {
|
||||
expectRefusal(parseMcpToolError(reply).error, kind === "large");
|
||||
if (operation === "extract_artifact")
|
||||
await expect(access(outputRoot)).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
} else {
|
||||
expect(reply.isError, JSON.stringify(reply)).not.toBe(true);
|
||||
if (operation === "extract_artifact")
|
||||
expect(
|
||||
(await readFile(join(outputRoot, "member.txt"))).equals(content),
|
||||
).toBe(true);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
expect((await call("close_binary", {})).isError).not.toBe(true);
|
||||
}
|
||||
expect((await readFile(path)).equals(bytes)).toBe(true);
|
||||
},
|
||||
);
|
||||
Vendored
+11
@@ -1,5 +1,16 @@
|
||||
import { crc32, zstdCompressSync } from "node:zlib";
|
||||
|
||||
/** A valid independent raw-block frame; its window is unrelated to its tiny output. */
|
||||
export const zstandardWindowFrameFixture = (windowDescriptor: number) => {
|
||||
const content = Buffer.from("REA independent frame control\n");
|
||||
const compressed = Buffer.from(
|
||||
"28b52ffd0090f1000052454120696e646570656e64656e74206672616d6520636f6e74726f6c0a",
|
||||
"hex",
|
||||
);
|
||||
compressed[5] = windowDescriptor;
|
||||
return { content, compressed };
|
||||
};
|
||||
|
||||
/** One non-Zip64 member, independently framed from the PKWARE APPNOTE records. */
|
||||
export const zstandardZipFixture = (
|
||||
content: Buffer,
|
||||
|
||||
Reference in new issue
Block a user