From 25aaf8928763c60f84c4d5bb4f369d976d2d6a92 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:32:52 +0800 Subject: [PATCH] fix(sqlite): budget schema names by returned JSON text Charge the whole-catalog admission guard by escaped decoded names, preserving raw-byte lookup identity and final worker reply limits. Keep UTF-16 inventory and detailed-schema results whose returned facts fit the documented allowance. Fixes #6888 --- docs/sqlite-databases.md | 3 ++ src/sqlite/SqliteDatabaseInspection.ts | 8 +++- tests/acceptance/sqliteDatabaseCli.test.ts | 44 ++++++++++++++++++ .../sqlite/sqliteDatabaseInspection.test.ts | 46 +++++++++++++++++++ tests/fixtures/sqlite/database.ts | 22 +++++++++ 5 files changed, 121 insertions(+), 2 deletions(-) diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index 3740320b99..f02712240e 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -128,6 +128,9 @@ bounds simultaneous input buffering and private snapshot/worker counts. The database and WAL together are limited to 256 MiB. SQLite values/records, SQL statements and the complete worker JSON reply each have a 16 MiB limit. +Schema names count by their returned JSON text, including escaping, for UTF-8, +UTF-16le and UTF-16be databases. Internal raw-byte lookup keys do not consume +the reply allowance; exact name identity still comes from those stored bytes. The owned worker has a 256 MiB JavaScript heap limit and defaults to a 30,000 ms deadline. Select optional MCP `worker_timeout_ms`, or CLI `--worker-timeout-ms `, to change the budget for that call. Values diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index a3d9b2c78c..3ea73c5f3b 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -324,7 +324,7 @@ const readSchema = ( // Names bound eventual output only when the whole catalog is requested. // Selected scope retains one table and at most one possible shadow parent; // unrelated names are examined without retaining them or charging the reply. - const metadataBudget = new OutputBudget(); + const catalogNameBudget = new OutputBudget(); for (const raw of db .prepare( "SELECT schema, name AS display_name, CAST(name AS BLOB) AS name, type, ncol, wr, strict FROM temp.pragma_table_list WHERE schema = 'main'", @@ -333,7 +333,11 @@ const readSchema = ( const value = tableRow.parse(raw); const nameKey = sqliteNameBytesKey(value.name); if (selectedName === undefined) { - metadataBudget.retain(nameKey); + // The private hex key grows with the database encoding, but is never + // returned. Admit catalog names by their decoded JSON representation; + // display text is used only for this lower bound, never for identity. + // Stored schema bytes still undergo strict decoding below. + catalogNameBudget.retain(value.display_name); metadata.set(nameKey, value); } else if (nameKey === selectedKey) { metadata.set(nameKey, value); diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index b58dad8775..83b129c77e 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -21,8 +21,11 @@ import { import { createSqliteApplicationCollationFixture, + createSqliteLongNameFixture, SQLITE_NATIVE_LIMITS_AVAILABLE, } from "../fixtures/sqlite/database.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { sqliteDatabaseSchema } from "../../src/domain/sqlite/sqliteDatabase.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; @@ -1137,6 +1140,47 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, ); +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "returns complete UTF-16 schemas through the worker and public CLI when JSON fits", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-large-catalog-"); + for (const [encoding, scope] of [ + ["UTF-16le", "inventory"], + ["UTF-16be", "database"], + ] as const) { + const { path, names } = createSqliteLongNameFixture(root, encoding); + const response = await runFileCli(cli, { + arguments: [ + "inspect-sqlite-database", + path, + "--schema-scope", + scope, + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence(response.json); + const report = sqliteDatabaseSchema.parse(evidence.normalized_result); + expect(evidence.parameters.schema_scope).toBe(scope); + expect(report.database_encoding).toBe(encoding); + expect(report.schema).toMatchObject({ scope, completeness: "complete" }); + expect(report.schema.tables).toHaveLength(names.length); + expect(report.schema.tables.map((table) => table.name)).toEqual( + expect.arrayContaining(names), + ); + expect( + report.schema.tables.every( + (table) => + table.columns_completeness === + (scope === "inventory" ? "not_requested" : "complete"), + ), + ).toBe(true); + expect(report.rows).toBeNull(); + } + }, +); + cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "keeps selected table metadata tied to exact schema-name bytes through CLI and MCP", async ({ cli }) => { diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index f30e311ab1..8044a9ada4 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -3,10 +3,12 @@ import { readFile, writeFile } from "node:fs/promises"; import { expect, it as nativeTest, onTestFinished } from "vitest"; import { createSqliteApplicationCollationFixture, + createSqliteLongNameFixture, SQLITE_NATIVE_LIMITS_AVAILABLE, } from "../../fixtures/sqlite/database.js"; import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js"; import { SqliteInspectionFailure } from "../../../src/sqlite/SqliteDatabaseLimits.js"; +import { jsonByteLength } from "../../../src/domain/jsonSerialization.js"; import { createTestWorkspace, removeTestWorkspace, @@ -238,6 +240,50 @@ it("counts schema names once when enforcing the complete reply budget", async () ); }); +it.each(["UTF-16le", "UTF-16be"] as const)( + "budgets %s catalog names by returned JSON rather than internal hex identity", + async (encoding) => { + const workspace = await createTestWorkspace("rea-sqlite-catalog-names-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const { path, names } = createSqliteLongNameFixture( + workspace.root, + encoding, + ); + for (const schema_scope of ["inventory", "database"] as const) { + const inspection = inspectSqliteDatabaseSnapshot(path, { + path, + schema_scope, + }); + expect(inspection.database_encoding).toBe(encoding); + expect(inspection.schema).toMatchObject({ + scope: schema_scope, + completeness: "complete", + }); + expect(inspection.schema.tables).toHaveLength(names.length); + expect(inspection.schema.tables.map((table) => table.name)).toEqual( + expect.arrayContaining(names), + ); + for (const table of inspection.schema.tables) { + expect(table.columns_completeness).toBe( + schema_scope === "inventory" ? "not_requested" : "complete", + ); + if (schema_scope === "database") { + expect(table.sql).toBe(`CREATE TABLE "${table.name}"(value TEXT)`); + expect(table.columns).toMatchObject([ + { name: "value", declared_type: "TEXT" }, + ]); + } else { + expect(table.sql).toBeUndefined(); + expect(table.columns).toEqual([]); + } + } + expect(jsonByteLength({ ok: true, inspection })).toBeLessThan( + 16 * 1024 * 1024, + ); + } + }, +); + it("keeps unrelated catalog names outside the selected-table reply budget", async () => { const path = await createDatabase( "CREATE TABLE selected(value TEXT); INSERT INTO selected VALUES ('retained');", diff --git a/tests/fixtures/sqlite/database.ts b/tests/fixtures/sqlite/database.ts index 53773c8fe2..445ba6b458 100644 --- a/tests/fixtures/sqlite/database.ts +++ b/tests/fixtures/sqlite/database.ts @@ -100,3 +100,25 @@ export const createSqliteApplicationCollationFixture = ( } return path; }; + +/** UTF-16 hex lookup keys exceed 16 MiB while both public schema scopes fit. */ +export const createSqliteLongNameFixture = ( + root: string, + encoding: "UTF-8" | "UTF-16le" | "UTF-16be", +): { path: string; names: string[] } => { + const path = join(root, `large-schema-${encoding}.db`); + const names = Array.from( + { length: 34 }, + (_, index) => `${index}_${"x".repeat(128 * 1024)}`, + ); + const producer = new DatabaseSync(path); + try { + producer.exec(`PRAGMA encoding='${encoding}'; BEGIN`); + for (const name of names) + producer.exec(`CREATE TABLE "${name}"(value TEXT)`); + producer.exec("COMMIT"); + } finally { + producer.close(); + } + return { path, names }; +};