mirror of
https://github.com/morluto/rea.git
synced 2026-11-13 18:22:42 +00:00
perf(entropy): separate physical reads from report resolution (#6950)
* perf(entropy): separate physical reads from report blocks * test(entropy): type the injected positional short-read boundary * test(entropy): retain full catalog transport capacity in task probe * docs(perf): record real entropy task costs and limits * docs(perf): record latest-main entropy qualification
This commit is contained in:
1 parent
b869fa762a
commit
b5c42dfc2a
7 files changed
+1973
-7
No files matched your search
@@ -38,6 +38,70 @@ it("keeps whole-file identity and entropy while selecting unaligned blocks and a
|
||||
expect(result.value.confidence).toBe("observed");
|
||||
});
|
||||
|
||||
it.each([256, 4096, 65536])(
|
||||
"keeps %i-byte blocks relative to an unaligned window across short physical reads",
|
||||
async (blockSize) => {
|
||||
const root = await createTestTempDirectory("rea-entropy-short-reads-");
|
||||
const path = join(root, "bytes");
|
||||
const bytes = Buffer.from(
|
||||
Array.from({ length: 3 * 65536 + 256 }, (_, i) => i % 256),
|
||||
);
|
||||
await writeFile(path, bytes);
|
||||
let reads = 0;
|
||||
const service = new ByteEntropyService({
|
||||
stat,
|
||||
open: async (...args) => {
|
||||
const file = await open(...args);
|
||||
const read = file.read.bind(file);
|
||||
Object.defineProperty(file, "read", {
|
||||
value: async (
|
||||
buffer: Buffer,
|
||||
offset: number,
|
||||
length: number,
|
||||
position: number,
|
||||
) => {
|
||||
reads += 1;
|
||||
return read(buffer, offset, Math.min(length, 3571), position);
|
||||
},
|
||||
});
|
||||
return file;
|
||||
},
|
||||
});
|
||||
try {
|
||||
const result = await service.inspect({
|
||||
path,
|
||||
offset: 3,
|
||||
length: 2 * 65536 + 1,
|
||||
block_size: blockSize,
|
||||
});
|
||||
if (!result.ok) throw result.error;
|
||||
const report = byteEntropySchema.parse(result.value.normalized_result);
|
||||
const fullBlocks = (2 * 65536) / blockSize;
|
||||
expect(report.blocks).toEqual([
|
||||
...Array.from({ length: fullBlocks }, (_, i) => ({
|
||||
offset: 3 + i * blockSize,
|
||||
length: blockSize,
|
||||
entropy: 8,
|
||||
})),
|
||||
{ offset: 3 + 2 * 65536, length: 1, entropy: 0 },
|
||||
]);
|
||||
expect(report.runs).toEqual([
|
||||
{ offset: 3, length: 2 * 65536, classification: "high" },
|
||||
{ offset: 3 + 2 * 65536, length: 1, classification: "low" },
|
||||
]);
|
||||
expect(report.file_entropy).toBe(8);
|
||||
expect(report.artifact.sha256).toBe(
|
||||
createHash("sha256").update(bytes).digest("hex"),
|
||||
);
|
||||
// Even with 3,571-byte short reads, finer report resolution does not
|
||||
// require hundreds of asynchronous reads of this 193 KiB file.
|
||||
expect(reads).toBeLessThan(100);
|
||||
} finally {
|
||||
await service.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects missing files, non-files and windows beyond EOF, but accepts empty windows", async () => {
|
||||
const root = await createTestTempDirectory("rea-entropy-input-");
|
||||
const path = join(root, "empty");
|
||||
|
||||
@@ -241,7 +241,10 @@ export class ByteEntropyService {
|
||||
);
|
||||
const end = input.offset + length;
|
||||
const blocks: ByteEntropy["blocks"] = [];
|
||||
const buffer = Buffer.alloc(input.block_size);
|
||||
// Physical reads must not shrink with report resolution, including
|
||||
// outside the selected window. Every admitted power-of-two block
|
||||
// size divides this buffer, so only the window's final block is short.
|
||||
const buffer = Buffer.alloc(65_536);
|
||||
const counts = new Float64Array(256);
|
||||
const hash = createHash("sha256");
|
||||
let position = 0;
|
||||
@@ -250,7 +253,7 @@ export class ByteEntropyService {
|
||||
const selected = position >= input.offset && position < end;
|
||||
const boundary =
|
||||
position < input.offset ? input.offset : selected ? end : size;
|
||||
const wanted = Math.min(input.block_size, boundary - position);
|
||||
const wanted = Math.min(buffer.length, boundary - position);
|
||||
let filled = 0;
|
||||
while (filled < wanted) {
|
||||
assertArtifactReadActive(options?.signal);
|
||||
@@ -267,11 +270,18 @@ export class ByteEntropyService {
|
||||
hash.update(bytes);
|
||||
for (const byte of bytes) counts[byte] = (counts[byte] ?? 0) + 1;
|
||||
if (selected)
|
||||
blocks.push({
|
||||
offset: position,
|
||||
length: filled,
|
||||
entropy: computeBlockEntropy(bytes),
|
||||
});
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < filled;
|
||||
offset += input.block_size
|
||||
) {
|
||||
const block = bytes.subarray(offset, offset + input.block_size);
|
||||
blocks.push({
|
||||
offset: position + offset,
|
||||
length: block.length,
|
||||
entropy: computeBlockEntropy(block),
|
||||
});
|
||||
}
|
||||
position += filled;
|
||||
}
|
||||
if (
|
||||
|
||||
Reference in new issue
Block a user