From c6b62754b095a134d6d67877508fa34311a2eda2 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:04:16 +0800 Subject: [PATCH] perf(kuna): reuse sorted string inventory for exact lookups Reuse the validated string inventory's numeric address order for O(log n + matches) exact reads, preserving every encoding at the selected address without a second lookup index. Keep streaming validation, session cache lifetime, full listings, provenance and caller choices. Validation: complete latest-head CI passed. Original real Kuna 1.833 Linux correctness and warm-lookup measurements remain pinned to their recorded feature revisions in the PR. The latest local 300 ms export-deadline observation and unchanged 31-test provider recheck are explicitly documented there; no assertion or timeout was weakened. Refs #4695. --- docs/kuna-provider.md | 2 +- src/kuna/KunaStringInventory.ts | 20 +++++ src/kuna/KunaStringQueries.ts | 3 +- .../kuna/kunaStringInventory.test.ts | 81 ++++++++++++++++++- .../providers/kuna/kunaStrings.test.ts | 60 ++++++++++++-- 5 files changed, 158 insertions(+), 8 deletions(-) diff --git a/docs/kuna-provider.md b/docs/kuna-provider.md index e4214502d1..d0c72e54d1 100644 --- a/docs/kuna-provider.md +++ b/docs/kuna-provider.md @@ -68,7 +68,7 @@ Single-function subprocess output and each cached source facet share a 16 MiB bu Results keep exact u64 virtual addresses, decoded text, native encoding, terminator status and byte length including any terminator. Kuna scans mapped sections or segments for runs of at least five characters; unterminated fragments can include instruction bytes. `all` includes ASCII, UTF-8 and the engine's Latin-range UTF-16LE reading. Other text, including short strings, non-Latin wide text and UTF-16BE, can remain undiscovered. An empty result does not prove absence. -Use `search_strings` for the matching rows or `list_strings` with an exact address for that location. Literal and regex matching run in REA, preserving the existing case and ECMAScript Unicode semantics instead of substituting Kuna's different regex dialect. Procedure and string regex queries share one worker slot per session. Input projections are materialized only after admission, so queued queries retain no per-query inventory arrays. Close cancels and drains matching. Native metadata queries also share one admission slot, including references; a failed stop retains admission until cleanup is proven. +Use `search_strings` for the matching rows or `list_strings` with an exact address for that location. The validated inventory is sorted by numeric address and encoding. Once cached, exact-address reads use binary search and return every encoding at that address without scanning the full inventory or retaining another lookup index. Literal and regex matching run in REA, preserving the existing case and ECMAScript Unicode semantics instead of substituting Kuna's different regex dialect. Procedure and string regex queries share one worker slot per session. Input projections are materialized only after admission, so queued queries retain no per-query inventory arrays. Close cancels and drains matching. Native metadata queries also share one admission slot, including references; a failed stop retains admission until cleanup is proven. `raw_result.strings` reports the source JSON digest/size, scanned scope, extraction policy, complete inventory count and selected count without copying unrequested text into the answer. Omit the address from `list_strings` to retrieve the complete inventory as Evidence; normal session retention and bundle export apply, including transport-capacity errors for oversized inline replies. These operations preserve their complete-result contracts and do not add paging. Function-referenced string facets in `analyze_function` remain unavailable; call `xrefs` explicitly when references to one returned string are needed. diff --git a/src/kuna/KunaStringInventory.ts b/src/kuna/KunaStringInventory.ts index 978f5b270f..0b90ccb363 100644 --- a/src/kuna/KunaStringInventory.ts +++ b/src/kuna/KunaStringInventory.ts @@ -93,12 +93,32 @@ const validateStringFacts = (entry: z.infer): void => { export type KunaStringItem = z.infer; export interface KunaStringInventory { + /** Canonical address order, then encoding; identities are unique. */ readonly items: readonly KunaStringItem[]; readonly sha256: string; readonly bytes: number; readonly scanned: "sections" | "segments"; } +/** Locate every encoding at a canonical address without copying an index. */ +export const kunaStringsAtAddress = ( + inventory: KunaStringInventory, + address: string, +): KunaStringItem[] => { + const { items } = inventory; + let start = 0; + let end = items.length; + while (start < end) { + const middle = Math.floor((start + end) / 2); + if (compareKunaAddresses(items[middle]!.address, address) < 0) + start = middle + 1; + else end = middle; + } + end = start; + while (end < items.length && items[end]!.address === address) end += 1; + return items.slice(start, end); +}; + /** Validates `kuna strings --json` row by row as Kuna writes it. */ export class KunaStringInventoryReader extends KunaJsonDocumentReader { readonly #items: KunaStringItem[] = []; diff --git a/src/kuna/KunaStringQueries.ts b/src/kuna/KunaStringQueries.ts index 17b83c16ef..222f40fa21 100644 --- a/src/kuna/KunaStringQueries.ts +++ b/src/kuna/KunaStringQueries.ts @@ -17,6 +17,7 @@ import { kunaQueryFailureForCaller } from "./KunaQueryErrors.js"; import { KUNA_STRING_MIN_LENGTH } from "./KunaLimits.js"; import { KunaStringInventoryReader, + kunaStringsAtAddress, type KunaStringInventory, type KunaStringItem, } from "./KunaStringInventory.js"; @@ -76,7 +77,7 @@ export class KunaStringQueries { const available = inventory.value.items; let items: KunaStringItem[] | undefined; if (address !== undefined) - items = available.filter((item) => item.address === address); + items = kunaStringsAtAddress(inventory.value, address); if (query?.success === true) { if (query.data.mode === "regex") { const matched = await this.work.match({ diff --git a/tests/boundary/providers/kuna/kunaStringInventory.test.ts b/tests/boundary/providers/kuna/kunaStringInventory.test.ts index 4bc1cb40bf..813b6d3e1e 100644 --- a/tests/boundary/providers/kuna/kunaStringInventory.test.ts +++ b/tests/boundary/providers/kuna/kunaStringInventory.test.ts @@ -5,7 +5,10 @@ import * as v8 from "node:v8"; import { expect, it, vi } from "vitest"; -import { KunaStringInventoryReader } from "../../../../src/kuna/KunaStringInventory.js"; +import { + KunaStringInventoryReader, + kunaStringsAtAddress, +} from "../../../../src/kuna/KunaStringInventory.js"; vi.mock("node:v8", { spy: true }); @@ -107,6 +110,13 @@ it.each([ { strings: [{ ...row, functions: [{ name: "main" }] }] }, ], ["duplicate identity", { count: 2, strings: [row, row] }], + [ + "nonadjacent duplicate identity", + { + count: 3, + strings: [row, { ...row, address: "@16", address_hex: "0x10" }, row], + }, + ], ])( "rejects inconsistent %s before publishing any strings", async (_label, changes) => { @@ -118,6 +128,75 @@ it.each([ }, ); +it("indexes canonical u64 addresses and keeps each encoding at the same address", async () => { + const entry = ( + address: bigint, + encoding: "ascii" | "utf8" | "utf16" = "ascii", + ) => ({ + ...row, + address: `@${address}`, + address_hex: `0x${address.toString(16)}`, + text: "hello", + length: 5, + byte_length: encoding === "utf16" ? 12 : 6, + encoding, + }); + const rows = [ + entry(0xffffffffffffffffn), + entry(16n, "utf16"), + entry(15n), + entry(16n), + entry(0x20000000000001n, "utf8"), + entry(0n), + entry(16n, "utf8"), + entry(17n), + ]; + const { result } = await read( + wire({ ...document(), count: rows.length, strings: rows }), + 3, + ); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(result.value.items.map(({ address }) => address)).toEqual([ + "0x0", + "0xf", + "0x10", + "0x10", + "0x10", + "0x11", + "0x20000000000001", + "0xffffffffffffffff", + ]); + for (const address of [ + "0x0", + "0x1", + "0xe", + "0xf", + "0x10", + "0x11", + "0x12", + "0x20000000000000", + "0x20000000000001", + "0x20000000000002", + "0xffffffffffffffff", + ]) + expect(kunaStringsAtAddress(result.value, address)).toEqual( + result.value.items.filter((item) => item.address === address), + ); + expect( + kunaStringsAtAddress(result.value, "0x10").map( + (item) => item.string?.encoding, + ), + ).toEqual(["ascii", "utf16", "utf8"]); +}); + +it("looks up an empty validated inventory", async () => { + const { result } = await read(wire({ ...document(), count: 0, strings: [] })); + expect(result.ok).toBe(true); + if (!result.ok) throw result.error; + expect(kunaStringsAtAddress(result.value, "0x0")).toEqual([]); +}); + it.each([ ["truncated", wire(document()).slice(0, -2)], ["trailing", `${wire(document())} {}`], diff --git a/tests/boundary/providers/kuna/kunaStrings.test.ts b/tests/boundary/providers/kuna/kunaStrings.test.ts index ebb18a8b87..a5430dc760 100644 --- a/tests/boundary/providers/kuna/kunaStrings.test.ts +++ b/tests/boundary/providers/kuna/kunaStrings.test.ts @@ -96,12 +96,62 @@ it("shares a typed string scan across concurrent searches and exact address read } }); -it.each(["not-json", "miscounted"] as const)( - "does not publish or cache a %s inventory", - async (strings) => { - const { open, kuna } = await session("rea-kuna-strings-invalid-", { - strings, +it("returns every encoding at an exact address in numeric inventory order", async () => { + const { open, kuna } = await session("rea-kuna-string-address-", { + stringRows: [ + { address: 0x100, text: "later" }, + { address: 0x10, text: "hello", encoding: "utf8" }, + { address: 0xf, text: "first" }, + { address: 0x10, text: "hello", encoding: "utf16" }, + { address: 0x10, text: "hello" }, + ], + }); + const client = open(); + try { + const exact = await run(client, "list_strings", { address: "0X000010" }); + expect(exact.result).toMatchObject([ + { address: "0x10", string: { encoding: "ascii" } }, + { address: "0x10", string: { encoding: "utf16" } }, + { address: "0x10", string: { encoding: "utf8" } }, + ]); + expect(exact.rawResult).toMatchObject({ + strings: { inventory_count: 5, selected_count: 3 }, }); + const complete = await run(client, "list_strings"); + expect(complete.result).toMatchObject([ + { address: "0xf" }, + { address: "0x10" }, + { address: "0x10" }, + { address: "0x10" }, + { address: "0x100" }, + ]); + for (const address of ["0x0", "0x11", "0xffffffffffffffff"]) + expect((await run(client, "list_strings", { address })).result).toEqual( + [], + ); + expect(await scans(kuna)).toHaveLength(1); + } finally { + expect(await client.close()).toEqual({ ok: true, value: null }); + } +}); + +it.each([ + ["not-json", { strings: "not-json" }], + ["miscounted", { strings: "miscounted" }], + [ + "duplicate identity", + { + stringRows: [ + { address: 0x30, text: "hello" }, + { address: 0x10, text: "other" }, + { address: 0x30, text: "hello" }, + ], + }, + ], +] as const)( + "does not publish or cache a %s inventory", + async (_label, behavior) => { + const { open, kuna } = await session("rea-kuna-strings-invalid-", behavior); const client = open(); try { expect(