perf(entropy): separate physical reads from report resolution (#6950)

* perf(entropy): separate physical reads from report blocks

* test(entropy): type the injected positional short-read boundary

* test(entropy): retain full catalog transport capacity in task probe

* docs(perf): record real entropy task costs and limits

* docs(perf): record latest-main entropy qualification
This commit is contained in:
gensokyo-dev authored and GitHub committed 2026-10-24 13:16:08 +08:00
1 parent b869fa762a
commit b5c42dfc2a
7 files changed
+1973 -7

No files matched your search

@@ -38,6 +38,70 @@ it("keeps whole-file identity and entropy while selecting unaligned blocks and a
expect(result.value.confidence).toBe("observed");
});
it.each([256, 4096, 65536])(
"keeps %i-byte blocks relative to an unaligned window across short physical reads",
async (blockSize) => {
const root = await createTestTempDirectory("rea-entropy-short-reads-");
const path = join(root, "bytes");
const bytes = Buffer.from(
Array.from({ length: 3 * 65536 + 256 }, (_, i) => i % 256),
);
await writeFile(path, bytes);
let reads = 0;
const service = new ByteEntropyService({
stat,
open: async (...args) => {
const file = await open(...args);
const read = file.read.bind(file);
Object.defineProperty(file, "read", {
value: async (
buffer: Buffer,
offset: number,
length: number,
position: number,
) => {
reads += 1;
return read(buffer, offset, Math.min(length, 3571), position);
},
});
return file;
},
});
try {
const result = await service.inspect({
path,
offset: 3,
length: 2 * 65536 + 1,
block_size: blockSize,
});
if (!result.ok) throw result.error;
const report = byteEntropySchema.parse(result.value.normalized_result);
const fullBlocks = (2 * 65536) / blockSize;
expect(report.blocks).toEqual([
...Array.from({ length: fullBlocks }, (_, i) => ({
offset: 3 + i * blockSize,
length: blockSize,
entropy: 8,
})),
{ offset: 3 + 2 * 65536, length: 1, entropy: 0 },
]);
expect(report.runs).toEqual([
{ offset: 3, length: 2 * 65536, classification: "high" },
{ offset: 3 + 2 * 65536, length: 1, classification: "low" },
]);
expect(report.file_entropy).toBe(8);
expect(report.artifact.sha256).toBe(
createHash("sha256").update(bytes).digest("hex"),
);
// Even with 3,571-byte short reads, finer report resolution does not
// require hundreds of asynchronous reads of this 193 KiB file.
expect(reads).toBeLessThan(100);
} finally {
await service.close();
}
},
);
it("rejects missing files, non-files and windows beyond EOF, but accepts empty windows", async () => {
const root = await createTestTempDirectory("rea-entropy-input-");
const path = join(root, "empty");
@@ -241,7 +241,10 @@ export class ByteEntropyService {
);
const end = input.offset + length;
const blocks: ByteEntropy["blocks"] = [];
const buffer = Buffer.alloc(input.block_size);
// Physical reads must not shrink with report resolution, including
// outside the selected window. Every admitted power-of-two block
// size divides this buffer, so only the window's final block is short.
const buffer = Buffer.alloc(65_536);
const counts = new Float64Array(256);
const hash = createHash("sha256");
let position = 0;
@@ -250,7 +253,7 @@ export class ByteEntropyService {
const selected = position >= input.offset && position < end;
const boundary =
position < input.offset ? input.offset : selected ? end : size;
const wanted = Math.min(input.block_size, boundary - position);
const wanted = Math.min(buffer.length, boundary - position);
let filled = 0;
while (filled < wanted) {
assertArtifactReadActive(options?.signal);
@@ -267,11 +270,18 @@ export class ByteEntropyService {
hash.update(bytes);
for (const byte of bytes) counts[byte] = (counts[byte] ?? 0) + 1;
if (selected)
blocks.push({
offset: position,
length: filled,
entropy: computeBlockEntropy(bytes),
});
for (
let offset = 0;
offset < filled;
offset += input.block_size
) {
const block = bytes.subarray(offset, offset + input.block_size);
blocks.push({
offset: position + offset,
length: block.length,
entropy: computeBlockEntropy(block),
});
}
position += filled;
}
if (