diff --git a/docs/guides/data/sqlite-databases.md b/docs/guides/data/sqlite-databases.md index f089347bbd..184b7a56a5 100644 --- a/docs/guides/data/sqlite-databases.md +++ b/docs/guides/data/sqlite-databases.md @@ -90,9 +90,20 @@ column limit. The selected database is readable and is not reported as malformed; request fewer columns per call instead. `row_limit` requires `table`, defaults to 20 and accepts positive safe integers. Samples remain subject to the byte and worker resource limits below, regardless of row count. The result reports -the selected limit, returned count and whether more rows were observed. Row -order is unspecified; a truncated sample does not prove anything about omitted -records. View, virtual-table and shadow-table row reads are unsupported. +the selected limit, returned count and whether more rows were observed. + +Rows are read in a stated unique order reported as `rows.order`: `rowid` for +rowid tables, and `primary_key` for `WITHOUT ROWID` tables, reusing the +primary-key index's own direction and declared collation. `rows.row_offset` +reports the rows skipped before the window and `rows.next_row_offset` the offset +to request for the following page; it is `null` once the window reaches the end. +Pass `row_offset` (CLI `--row-offset`, requires `table`, default 0) to skip into +a table rather than repeating earlier rows. A table that declares all of +`rowid`, `_rowid_` and `oid` as columns and has no primary key reports +`order: "unspecified"` and refuses a non-zero `row_offset`, because the skipped +prefix would be arbitrary. Pages compose only while the reported database and +WAL SHA-256 match; re-inspecting a changed snapshot restarts the sequence. +View, virtual-table and shadow-table row reads are unsupported. SQLite integers are decimal strings so every signed 64-bit value survives JSON serialization. Real values retain their numbers; `"Infinity"`, `"-Infinity"` diff --git a/src/application/desktop/DesktopSqliteState.test.ts b/src/application/desktop/DesktopSqliteState.test.ts index 5b787e7589..4829b037f2 100644 --- a/src/application/desktop/DesktopSqliteState.test.ts +++ b/src/application/desktop/DesktopSqliteState.test.ts @@ -58,6 +58,8 @@ const report = (selected: string | undefined) => ({ returned_rows: 1, truncated: false, order: "unspecified", + row_offset: 0, + next_row_offset: null, }, limitations: [], }); diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts index 1be65ea4dd..5637ac53ef 100644 --- a/src/application/sqlite/SqliteDatabaseService.test.ts +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -120,7 +120,9 @@ it("accepts sqlite_schema rows without requiring a fabricated declaration", asyn row_limit: 20, returned_rows: 1, truncated: false, - order: "unspecified", + order: "rowid", + row_offset: 0, + next_row_offset: null, }, }; const service = new SqliteDatabaseService({ diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts index 6f79fbdd63..bb83c7c25b 100644 --- a/src/application/sqlite/SqliteDatabaseService.ts +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -59,6 +59,7 @@ const matchesSelectedDatabase = ( ? report.rows === null : report.rows?.table === input.table && report.rows.row_limit === (input.row_limit ?? SQLITE_ROW_LIMIT_DEFAULT) && + report.rows.row_offset === (input.row_offset ?? 0) && (input.columns === undefined || (report.rows.columns.length === input.columns.length && report.rows.columns.every( diff --git a/src/cli/sqliteCommands.ts b/src/cli/sqliteCommands.ts index 26fae79cd4..29265eb24a 100644 --- a/src/cli/sqliteCommands.ts +++ b/src/cli/sqliteCommands.ts @@ -48,6 +48,11 @@ export const registerSqliteCommands = ( rowLimit: cliDecimalField( inspectSqliteDatabaseInputSchema.shape.row_limit, ).describe("Positive safe integer row count; default 20; requires table"), + rowOffset: cliDecimalField( + inspectSqliteDatabaseInputSchema.shape.row_offset, + ).describe( + "Non-negative safe integer rows to skip before the window; default 0; requires table", + ), }), run: ({ args, options }) => withCommandCancellation((signal) => @@ -67,6 +72,9 @@ export const registerSqliteCommands = ( ...(options.rowLimit === undefined ? {} : { row_limit: options.rowLimit }), + ...(options.rowOffset === undefined + ? {} + : { row_offset: options.rowOffset }), ...(options.column === undefined ? {} : { columns: options.column }), diff --git a/src/contracts/sqlite/sqliteToolContracts.ts b/src/contracts/sqlite/sqliteToolContracts.ts index c1e9dfcf87..59a8ed617a 100644 --- a/src/contracts/sqlite/sqliteToolContracts.ts +++ b/src/contracts/sqlite/sqliteToolContracts.ts @@ -14,7 +14,7 @@ export const SQLITE_TOOL_CONTRACTS = [ kind: "artifact-provider", requiresActiveTarget: true, description: - "Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns observed Evidence with database/WAL SHA-256 and a compact schema inventory. Selecting an exact ordinary table expands only its schema and samples rows (default 20, subject to byte and worker resource limits). Select exact columns in the requested order to avoid unrelated or large values; schema_scope=database explicitly expands the whole schema; returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. Adjust worker_timeout_ms (default 30000 ms) for worker startup, queries and reply writing; allow separate client headroom for snapshot acquisition, reply reads, cleanup and delivery. The result retains the actual budget; byte/heap safety limits stay fixed. No arbitrary SQL, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. At most two inspections per provider are admitted; occupied capacity is retryable. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.", + "Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns observed Evidence with database/WAL SHA-256 and a compact schema inventory. Selecting an exact ordinary table expands only its schema and samples rows (default 20, subject to byte and worker resource limits). Retained rows are read in a stated unique order (rowid, or the declared primary key for WITHOUT ROWID tables) and report row_offset/next_row_offset, so a caller can page with row_offset; pages compose only while the reported artifact and WAL SHA-256 match. A rowid table whose rowid, _rowid_ and oid are all shadowed by declared columns has no unique order to report: it returns order=unspecified with a null next_row_offset and refuses any non-zero row_offset and cannot be paged; raise row_limit for a larger first sample within the resource limits. Select exact columns in the requested order to avoid unrelated or large values; schema_scope=database explicitly expands the whole schema; returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. Adjust worker_timeout_ms (default 30000 ms) for worker startup, queries and reply writing; allow separate client headroom for snapshot acquisition, reply reads, cleanup and delivery. The result retains the actual budget; byte/heap safety limits stay fixed. No arbitrary SQL, ORDER BY or WHERE, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. At most two inspections per provider are admitted; occupied capacity is retryable. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.", inputSchema: inspectSqliteDatabaseInputSchema, outputSchema: evidenceResultOf(sqliteDatabaseSchema), examples: [ @@ -31,6 +31,15 @@ export const SQLITE_TOOL_CONTRACTS = [ row_limit: 20, }, }, + { + title: "Read a later page of selected rows in a stated order", + input: { + path: "/artifacts/application.sqlite", + table: "settings", + row_offset: 20, + row_limit: 20, + }, + }, { title: "Expand the complete stored schema", input: { diff --git a/src/domain/sqlite/sqliteDatabase.ts b/src/domain/sqlite/sqliteDatabase.ts index ece4d30d62..4d3fa929e5 100644 --- a/src/domain/sqlite/sqliteDatabase.ts +++ b/src/domain/sqlite/sqliteDatabase.ts @@ -60,6 +60,13 @@ export const inspectSqliteDatabaseInputSchema = z .describe( "Selected-table sample size, default 20; subject to byte and worker resource limits", ), + row_offset: z + .int() + .min(0) + .optional() + .describe( + "Rows to skip before the retained window; default 0; requires table", + ), }) .describe("When schema_scope is selected_table, also provide table.") .superRefine((input, context) => { @@ -69,6 +76,12 @@ export const inspectSqliteDatabaseInputSchema = z path: ["row_limit"], message: "row_limit requires an explicit table selection", }); + if (input.row_offset !== undefined && input.table === undefined) + context.addIssue({ + code: "custom", + path: ["row_offset"], + message: "row_offset requires an explicit table selection", + }); if (input.columns !== undefined && input.table === undefined) context.addIssue({ code: "custom", @@ -83,7 +96,11 @@ export const inspectSqliteDatabaseInputSchema = z }); }) .meta({ - dependentRequired: { row_limit: ["table"], columns: ["table"] }, + dependentRequired: { + row_limit: ["table"], + columns: ["table"], + row_offset: ["table"], + }, if: { properties: { schema_scope: { const: "selected_table" } }, required: ["schema_scope"], @@ -199,7 +216,19 @@ export const sqliteDatabaseSchema = z row_limit: z.int().min(1), returned_rows: nonnegative, truncated: z.boolean(), - order: z.literal("unspecified"), + order: z + .enum(["rowid", "primary_key", "unspecified"]) + .describe( + "Unique order the retained window is read in; unspecified means the table shadows rowid, _rowid_ and oid and no primary key is declared", + ), + row_offset: nonnegative.describe( + "Rows skipped before this window; 0 for the sample taken without a selection", + ), + next_row_offset: nonnegative + .nullable() + .describe( + "Offset to request for the following page; null when the sampled window reaches the end or the table has no stated order. Pages compose only while artifact.sha256 and wal.sha256 match", + ), }) .nullable(), limitations: z.array(z.string()), @@ -210,13 +239,17 @@ export const sqliteDatabaseSchema = z rows !== null && (rows.returned_rows !== rows.values.length || rows.returned_rows > rows.row_limit || - rows.values.some((row) => row.length !== rows.columns.length)) + rows.values.some((row) => row.length !== rows.columns.length) || + rows.next_row_offset !== + (rows.truncated && rows.order !== "unspecified" + ? rows.row_offset + rows.returned_rows + : null)) ) context.addIssue({ code: "custom", path: ["rows"], message: - "Selected row counts and column dimensions must match the retained values", + "Selected row counts, column dimensions and the next window must match the retained values", }); }); @@ -240,6 +273,7 @@ export const sqliteInspectionParameters = ( ? {} : { schema_scope: input.schema_scope }), ...(input.row_limit === undefined ? {} : { row_limit: input.row_limit }), + ...(input.row_offset === undefined ? {} : { row_offset: input.row_offset }), }); /** One contract owner interprets omitted scope for adapters and selection checks. */ diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index 3ea73c5f3b..596f8269fa 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -19,6 +19,8 @@ import { /** SQLite's generic SQL error, raised while compiling or evaluating an expression. */ const SQLITE_ERROR = 1; +/** SQLite refuses statement text or a record above a fixed size with this result code. */ +const SQLITE_TOOBIG = 18; const natural = z .bigint() @@ -72,8 +74,11 @@ type Inspection = Pick< >; type SelectedTableFacts = Pick< SqliteDatabase["schema"]["tables"][number], - "name" | "kind" | "columns" ->; + "name" | "kind" | "columns" | "without_rowid" +> & { + /** Absent for a non-ordinary table, whose row facts are never probed. */ + readonly orderKey?: RowKeyFacts; +}; class OutputBudget { #bytes = 1024; @@ -284,6 +289,124 @@ const readColumns = ( return columns.sort((left, right) => left.cid - right.cid); }; +/** + * The facts a row window needs and the caller's projection cannot supply: which + * rowid alias spellings the table declares, and the primary key that orders a + * WITHOUT ROWID table. Alias detection compares raw name bytes against ASCII, so + * an unrelated malformed identifier is never decoded on the way. + */ +type RowKeyFacts = { + readonly withoutRowid: boolean; + readonly primaryKey: readonly { + readonly name: string; + readonly position: number; + }[]; + readonly shadowedAliases: ReadonlySet; +}; + +const ROWID_ALIASES = ["rowid", "_rowid_", "oid"] as const; + +/** Name bytes per character for each encoding `asciiLower` accepts. */ +const ASCII_CHARACTER_BYTES: Readonly> = { + "UTF-8": 1, + "UTF-16le": 2, + "UTF-16be": 2, +}; + +const asciiLower = ( + bytes: Uint8Array, + encoding: DatabaseEncoding, +): string | undefined => { + if (encoding === "UTF-8") { + let text = ""; + for (const byte of bytes) { + if (byte >= 0x80) return undefined; + text += String.fromCharCode( + byte >= 0x41 && byte <= 0x5a ? byte + 0x20 : byte, + ); + } + return text; + } + if (bytes.length % 2 !== 0) return undefined; + const lowFirst = encoding === "UTF-16le"; + let text = ""; + for (let index = 0; index < bytes.length; index += 2) { + const unit = lowFirst + ? bytes[index]! | (bytes[index + 1]! << 8) + : (bytes[index]! << 8) | bytes[index + 1]!; + if (unit >= 0x80) return undefined; + text += String.fromCharCode( + unit >= 0x41 && unit <= 0x5a ? unit + 0x20 : unit, + ); + } + return text; +}; + +const readRowKeyFacts = ( + db: DatabaseSync, + name: string, + encoding: DatabaseEncoding, +): RowKeyFacts => { + // SQLite enables up to 2 KiB names by default, but a rebuilt table can declare + // more; an alias test that decodes a name first would turn that into an + // unbounded allocation. Alias detection only needs names of alias byte length. + const maximumAliasBytes = + Math.max(...ROWID_ALIASES.map((alias) => alias.length)) * + ASCII_CHARACTER_BYTES[encoding]; + + const row = z + .object({ wr: z.union([z.bigint(), z.number()]) }) + .optional() + .parse( + db + .prepare( + "SELECT wr FROM temp.pragma_table_list WHERE schema = 'main' AND name = ?", + ) + .get(name) ?? undefined, + ); + const withoutRowid = row !== undefined && row.wr !== 0 && row.wr !== 0n; + const shadowed = new Set(); + const primaryKey: { name: string; position: number }[] = []; + for (const raw of db + .prepare( + "SELECT cid, CAST(name AS BLOB) AS name, pk FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 2, 3) ORDER BY cid", + ) + .iterate(name)) { + const value = z + .object({ + cid: z.union([z.bigint(), z.number()]), + name: z.instanceof(Uint8Array), + pk: z.union([z.bigint(), z.number()]), + }) + .parse(raw); + const alias = + value.name.length <= maximumAliasBytes + ? asciiLower(value.name, encoding) + : undefined; + if ( + alias !== undefined && + (ROWID_ALIASES as readonly string[]).includes(alias) + ) + shadowed.add(alias); + // A rowid table's ORDER BY names an alias, never a declared key, so only a + // WITHOUT ROWID table's key columns are worth decoding — and a malformed + // unselected key name must not abort an otherwise readable selection. + const pk = Number(value.pk); + if (withoutRowid && pk > 0) + primaryKey.push({ + name: decodeSchemaText(value.name, encoding, "primary key column"), + position: pk, + }); + } + return { + withoutRowid, + primaryKey: primaryKey.sort( + (left, right) => left.position - right.position, + ), + shadowedAliases: shadowed, + }; +}; + const readSchema = ( db: DatabaseSync, input: InspectSqliteDatabaseInput, @@ -430,6 +553,8 @@ const readSchema = ( encoding, selectedColumns: input.columns, }), + without_rowid: selectedRowSource.wr !== 0n, + orderKey: readRowKeyFacts(db, selectedRowSourceName, encoding), }; // NOCASE folds only ASCII letters, as SQLite does when it resolves the // table named by an index or trigger. Fold only for an ordinary table whose @@ -513,8 +638,20 @@ const readSchema = ( encoding, selectedColumns: scope === "inventory" ? input.columns : undefined, }); + // The window's key comes from the table, not the caller's projection: + // an inventory request may select columns that omit the primary key, + // and the order must still be discoverable. Read only the key columns + // so an unrelated malformed name is not decoded on the way. if (input.table === value.name) - selectedTable = { name: table.name, kind: table.kind, columns }; + selectedTable = { + name: table.name, + kind: table.kind, + columns, + without_rowid: table.without_rowid, + ...(table.kind === "table" + ? { orderKey: readRowKeyFacts(db, value.name, encoding) } + : {}), + }; if (scope !== "inventory") table.columns = columns; } if (input.table === value.name && selectedTable === undefined) @@ -522,6 +659,10 @@ const readSchema = ( name: table.name, kind: table.kind, columns: table.columns, + without_rowid: table.without_rowid, + ...(table.kind === "table" + ? { orderKey: readRowKeyFacts(db, value.name, encoding) } + : {}), }; schema.tables.push(table); } else if (value.type === "index") { @@ -593,7 +734,11 @@ const resultColumnLimitFailure = ( cause: unknown, selectedColumns: number, ): SqliteColumnLimitFailure | undefined => { - if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return undefined; + // A statement wide enough to overflow SQLite's SQL-text length fails as + // SQLITE_TOOBIG before it can report the result-column limit, but the + // projection is still the reason it did. + const code = sqlitePrimaryResultCode(cause); + if (code !== SQLITE_ERROR && code !== SQLITE_TOOBIG) return undefined; const engineColumnLimit = db.limits.column; if ( !Number.isSafeInteger(engineColumnLimit) || @@ -622,11 +767,16 @@ const rowEvaluationFailure = ( readonly table: SelectedTableFacts; readonly columns: readonly string[]; readonly source: string; + readonly clause: string; readonly limit: number; + readonly offset: number; readonly encoding: DatabaseEncoding; }, ): unknown => { - const { table, columns, source, limit, encoding } = selection; + const { table, columns, source, clause, limit, offset, encoding } = selection; + // Probes must evaluate the same window the failed statement asked for, or a + // failure confined to a later page is attributed to the wrong rows. + const window = `${source}${clause}`; const columnLimit = resultColumnLimitFailure(db, cause, columns.length); if (columnLimit !== undefined) return columnLimit; if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return cause; @@ -644,9 +794,9 @@ const rowEvaluationFailure = ( // A WITHOUT ROWID source may require an unavailable application collation // even for a constant projection. Do not attribute that failure to every // generated column or recommend other columns from the same unreadable source. - db.prepare(`SELECT count(*) FROM (SELECT 1 FROM ${source} LIMIT ?)`).get( - limit, - ); + db.prepare( + `SELECT count(*) FROM (SELECT 1 FROM ${window} LIMIT ? OFFSET ?)`, + ).get(limit, offset); } catch (failure: unknown) { return sqlitePrimaryResultCode(failure) === SQLITE_ERROR ? unavailable @@ -656,8 +806,8 @@ const rowEvaluationFailure = ( const probe = (name: string): { readonly failure: unknown } | undefined => { try { db.prepare( - `SELECT count(${cellByteLength(name)}) FROM (SELECT ${quoteIdentifier(name)} FROM ${source} LIMIT ?)`, - ).get(limit); + `SELECT count(${cellByteLength(name)}) FROM (SELECT ${quoteIdentifier(name)} FROM ${window} LIMIT ? OFFSET ?)`, + ).get(limit, offset); return undefined; } catch (failure: unknown) { return { failure }; @@ -700,6 +850,84 @@ const rowEvaluationFailure = ( ); }; +type RowOrder = { + readonly order: "rowid" | "primary_key" | "unspecified"; + readonly clause: string; +}; + +const indexKeyRow = z.object({ + name: z.instanceof(Uint8Array), + desc: z.union([z.bigint(), z.number()]), + coll: z.instanceof(Uint8Array), +}); + +/** + * Choose the unique order a window is read in, without accepting caller SQL. + * A rowid table exposes an implicit rowid unless all three spellings are + * declared columns. A WITHOUT ROWID table orders by its primary key, reusing the + * index's own direction and collation so no sorter is introduced. + */ +const readRowOrder = ( + db: DatabaseSync, + table: SelectedTableFacts, + encoding: DatabaseEncoding, +): RowOrder => { + const facts = table.orderKey; + // A non-ordinary table's keys are not probed, and none of its rows are read. + if (facts === undefined) return { order: "unspecified", clause: "" }; + if (!table.without_rowid) { + // ORDER BY must name an alias SQLite still binds: a declared `rowid` column + // shadows that spelling and would sort by a stored column that need not be + // unique. Pick the first spelling the table leaves free. + const alias = ROWID_ALIASES.find( + (spelling) => !facts.shadowedAliases.has(spelling), + ); + return alias === undefined + ? { order: "unspecified", clause: "" } + : { order: "rowid", clause: ` ORDER BY ${alias}` }; + } + const key = facts.primaryKey; + if (key.length === 0) return { order: "unspecified", clause: "" }; + const primaryKey = db + .prepare( + "SELECT CAST(name AS BLOB) AS name FROM temp.pragma_index_list(?) WHERE origin = 'pk'", + ) + .get(table.name) as { readonly name: unknown } | undefined; + if (primaryKey === undefined) return { order: "unspecified", clause: "" }; + // A WITHOUT ROWID key may repeat a column at different collations or + // directions — `PRIMARY KEY(a COLLATE BINARY, a COLLATE NOCASE DESC)` is + // valid — so the terms must follow the index's own sequence. Keying these + // by column name would collapse the repetition and advertise an order the + // actual key does not establish. + const ordered = new Map< + string, + { readonly desc: boolean; readonly coll: string } + >(); + const terms: string[] = []; + for (const raw of db + .prepare( + 'SELECT CAST(name AS BLOB) AS name, "desc", CAST(coll AS BLOB) AS coll FROM temp.pragma_index_xinfo(CAST(? AS BLOB)) WHERE key = 1 ORDER BY seqno', + ) + .iterate(primaryKey.name as Uint8Array) as Iterable) { + const { name, desc, coll } = indexKeyRow.parse(raw); + const column = decodeSchemaText(name, encoding, "index column"); + const detail = { + desc: desc === 1n || desc === 1, + coll: decodeSchemaText(coll, encoding, "collation"), + }; + ordered.set(column, detail); + // The declared collation keeps NOCASE/RTRIM keys in index order; spelling + // it explicitly also prevents a caller column's own collation from + // re-sorting. + terms.push( + `${quoteIdentifier(column)} COLLATE ${quoteIdentifier(detail.coll)}${detail.desc ? " DESC" : ""}`, + ); + } + for (const column of key) + if (!ordered.has(column.name)) return { order: "unspecified", clause: "" }; + return { order: "primary_key", clause: ` ORDER BY ${terms.join(", ")}` }; +}; + const readRows = ( db: DatabaseSync, input: InspectSqliteDatabaseInput, @@ -723,6 +951,7 @@ const readRows = ( `Rows from ${table.kind} table ${input.table} are outside the ordinary-table inspection profile.`, ); const limit = input.row_limit ?? SQLITE_ROW_LIMIT_DEFAULT; + const offset = input.row_offset ?? 0; const visibleColumns = table.columns .filter((column) => column.hidden !== 1) .map((column) => column.name); @@ -741,6 +970,15 @@ const readRows = ( }) .join(", "); const source = `main.${quoteIdentifier(input.table)} NOT INDEXED`; + const { order, clause } = readRowOrder(db, table, encoding); + // An offset needs a stated order to mean anything; refuse instead of skipping + // an arbitrary prefix of an order the reply cannot claim. + if (offset > 0 && order === "unspecified") + throw new SqliteInspectionFailure( + "row-offset", + `row_offset requires a stated row order, but table ${input.table} shadows rowid, _rowid_ and oid and declares no primary key. Read the table without row_offset.`, + ); + const window = `${source}${clause} LIMIT ? OFFSET ?`; const rows: NonNullable = { table: input.table, columns, @@ -748,19 +986,21 @@ const readRows = ( row_limit: limit, returned_rows: 0, truncated: false, - order: "unspecified", + order, + row_offset: offset, + next_row_offset: null, }; // Keep query-source, expression, capacity and malformed-data failures distinct. try { const lengths = db.prepare( - `SELECT ${columns.map(cellByteLength).join(", ")} FROM ${source} LIMIT ?`, + `SELECT ${columns.map(cellByteLength).join(", ")} FROM ${window}`, ); lengths.setReturnArrays(true); - for (const raw of lengths.iterate(limit)) + for (const raw of lengths.iterate(limit, offset)) budget.beforeProjection(cellByteLengths.parse(raw)); - const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`); + const statement = db.prepare(`SELECT ${projection} FROM ${window}`); statement.setReturnArrays(true); - for (const raw of statement.iterate(limit)) { + for (const raw of statement.iterate(limit, offset)) { const rawRow = arrayRow.parse(raw); // Hex expansion already belongs to the retained-output budget, before decoding/copying cells. budget.beforeDecode(rawRow); @@ -769,18 +1009,28 @@ const readRows = ( rows.values.push(values); } rows.truncated = - db.prepare(`SELECT 1 FROM ${source} LIMIT 1 OFFSET ?`).get(limit) !== - undefined; + db + .prepare(`SELECT 1 FROM ${source}${clause} LIMIT 1 OFFSET ?`) + .get(offset + rows.values.length) !== undefined; } catch (cause: unknown) { throw rowEvaluationFailure(db, cause, { table, columns, source, + clause, limit, + offset, encoding, }); } rows.returned_rows = rows.values.length; + // Only an ordered window with rows beyond it can be continued by another + // page, and row_offset refuses any non-zero offset on an unordered table, so + // publishing one there would name a request that is always refused. + rows.next_row_offset = + rows.truncated && order !== "unspecified" + ? offset + rows.returned_rows + : null; return rows; }; @@ -827,7 +1077,7 @@ export const inspectSqliteDatabaseSnapshot = ( "The source database and WAL are never opened by SQLite or modified. A private copy may acquire its own WAL-index sidecar and is removed after inspection.", "SQLite reports current schema and selected committed rows from its valid database/WAL state. Deleted records, uncommitted or invalid WAL tails, encrypted databases, forensic recovery, and full integrity verification are outside this profile.", "Stored CREATE statements preserve index/view/trigger definitions. Views, triggers, and virtual/shadow table rows are not executed; virtual-table column expansion is unsupported.", - "Selected rows have unspecified order and are a bounded sample. Text display may replace invalid sequences; bytes_hex and encoding preserve SQLite's text byte representation, not original on-disk page offsets.", + "Selected rows are a bounded sample; rows.order names the order they were read in, and rows.next_row_offset is null either when the window reaches the end or when rows.order is unspecified, so a truncated sample with no stated order cannot be paged further. Text display may replace invalid sequences; bytes_hex and encoding preserve SQLite's text byte representation, not original on-disk page offsets.", "Resource boundaries: at most two concurrent inspections per provider; combined input 256 MiB per inspection, complete worker JSON 16 MiB, native SQLite value/record and SQL statement lengths 16 MiB, owned-worker JavaScript heap 256 MiB. The parent applies a separate supervised worker deadline, reported in worker_timeout_ms. These are distinct from total resident memory; an overall native SQLite heap cap is not claimed. Failures return no partial success.", ]; const budget = new OutputBudget(); diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 56f25d8666..2a118aadc7 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -5,20 +5,34 @@ export const SQLITE_DATABASE_LIMITS = { outputBytes: 16 * 1024 * 1024, diagnosticBytes: 1024 * 1024, } as const; +/** + * Failure reasons the worker protocol may carry. The reply schema and the + * worker's own failure type read this one list, so a refusal that the worker + * can emit cannot be one the client refuses to parse. + */ +export const SQLITE_INSPECTION_REASONS = [ + "format", + "selection", + "column-selection", + "row-offset", + "column-evaluation", + "row-evaluation", + "column-limit", + "unsupported", + "unavailable", + "output-limit", + "memory", +] as const; +export type SqliteInspectionReason = (typeof SQLITE_INSPECTION_REASONS)[number]; +/** Reasons that carry their own reply fields rather than a bare message. */ +export const SQLITE_STRUCTURED_INSPECTION_REASONS = [ + "column-evaluation", + "column-limit", +] as const; /** Internal rejection retains the failed boundary without returning partial success. */ export class SqliteInspectionFailure extends Error { constructor( - readonly reason: - | "format" - | "selection" - | "column-selection" - | "column-evaluation" - | "row-evaluation" - | "column-limit" - | "unsupported" - | "unavailable" - | "output-limit" - | "memory", + readonly reason: SqliteInspectionReason, message: string, ) { super(message); diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index 08e8b5b218..f47188eebe 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -37,7 +37,11 @@ import type { ProviderProcessStopResult, } from "../process/ProviderProcess.js"; import { capturedSqliteOutput } from "./SqliteDatabaseFailures.js"; -import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; +import { + SQLITE_DATABASE_LIMITS, + SQLITE_INSPECTION_REASONS, + SQLITE_STRUCTURED_INSPECTION_REASONS, +} from "./SqliteDatabaseLimits.js"; const operation = "inspect_sqlite_database"; const { @@ -51,16 +55,9 @@ const replySchema = z.union([ z.strictObject({ ok: z.literal(true), inspection: inspectionSchema }), z.strictObject({ ok: z.literal(false), - reason: z.enum([ - "format", - "selection", - "column-selection", - "row-evaluation", - "unsupported", - "unavailable", - "output-limit", - "memory", - ]), + reason: z + .enum(SQLITE_INSPECTION_REASONS) + .exclude(SQLITE_STRUCTURED_INSPECTION_REASONS), message: z.string(), }), z.strictObject({ @@ -94,6 +91,7 @@ const rejectedArgument = { unsupported: "path", selection: "table", "column-selection": "columns", + "row-offset": "row_offset", } as const; const rejectedReply = ( diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index 3308c15681..7c62a4f80e 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -48,6 +48,108 @@ const parityMcp = createSharedMcpSession(async (registerCleanup, isClosed) => { }); afterAll(() => parityMcp.close()); +/** Run the offset page through the CLI so the option name is covered as well as the MCP input. */ +const pagedCliRows = async ( + cli: Parameters[0], + path: string, +): Promise => { + const result = await runFileCli(cli, { + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "selected", + "--column", + "id", + "--row-offset", + "1", + "--row-limit", + "1", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + return result.json; +}; + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "pages selected rows in a stated order identically through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-page-"); + const path = join(root, "selected.db"); + const database = new DatabaseSync(path); + try { + database.exec(` + CREATE TABLE selected(id INTEGER, label TEXT); + INSERT INTO selected(id, label) VALUES (1, 'first'), (2, 'second'), (3, 'third'); + CREATE TABLE keyed(id INTEGER, label TEXT, PRIMARY KEY(id DESC)) WITHOUT ROWID; + INSERT INTO keyed(id, label) VALUES (1, 'low'), (2, 'mid'), (3, 'high');`); + } finally { + database.close(); + } + const { call } = await connectLocalToolsMcp(); + const expected = { + parameters: { row_offset: 1, row_limit: 1 }, + normalized_result: { + rows: { + columns: ["id"], + order: "rowid", + row_offset: 1, + row_limit: 1, + returned_rows: 1, + next_row_offset: 2, + values: [[{ type: "integer", value: "2" }]], + }, + }, + }; + const paged = await call("inspect_sqlite_database", { + path, + table: "selected", + columns: ["id"], + row_offset: 1, + row_limit: 1, + }); + expect(paged.isError).not.toBe(true); + expect(paged.structuredContent).toMatchObject(expected); + expect(await pagedCliRows(cli, path)).toMatchObject(expected); + const primaryKey = await call("inspect_sqlite_database", { + path, + table: "keyed", + columns: ["id"], + }); + expect(primaryKey.isError).not.toBe(true); + expect(primaryKey.structuredContent).toMatchObject({ + normalized_result: { + rows: { + order: "primary_key", + values: [ + [{ type: "integer", value: "3" }], + [{ type: "integer", value: "2" }], + [{ type: "integer", value: "1" }], + ], + }, + }, + }); + const refused = await call("inspect_sqlite_database", { + path, + row_offset: 1, + }); + expect(parseMcpToolError(refused).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "inspect_sqlite_database", + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["row_offset"], + reason: "invalid_value", + }), + ]), + }, + }); + }, +); + cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "unions SQLite cell slices with numeric cell paths", async ({ cli }) => { diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index 44bdaa1b80..aefc8e97c6 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -681,6 +681,283 @@ it("samples above 1000 rows and safe integer limits within the byte budget", asy } }); +it("pages a rowid table in rowid order without repeating or dropping rows", async () => { + const path = await createDatabase(`CREATE TABLE items(value TEXT); + INSERT INTO items(value) VALUES ('a'), ('b'), ('c'), ('d'), ('e');`); + const first = inspectSqliteDatabaseSnapshot(path, { + path, + table: "items", + row_limit: 2, + }); + expect(first.rows).toMatchObject({ + order: "rowid", + row_offset: 0, + returned_rows: 2, + truncated: true, + next_row_offset: 2, + }); + const second = inspectSqliteDatabaseSnapshot(path, { + path, + table: "items", + row_limit: 2, + row_offset: first.rows?.next_row_offset ?? 0, + }); + expect(second.rows).toMatchObject({ + order: "rowid", + row_offset: 2, + returned_rows: 2, + truncated: true, + next_row_offset: 4, + }); + const last = inspectSqliteDatabaseSnapshot(path, { + path, + table: "items", + row_limit: 2, + row_offset: second.rows?.next_row_offset ?? 0, + }); + expect(last.rows).toMatchObject({ + row_offset: 4, + returned_rows: 1, + truncated: false, + next_row_offset: null, + }); + // Reading every page once must reproduce the table exactly: asserting only + // counts and offsets would pass even if a page repeated the first rows. + expect( + [first, second, last].flatMap((page) => + (page.rows?.values ?? []).map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ), + ).toEqual([["a"], ["b"], ["c"], ["d"], ["e"]]); +}); + +it.each(["UTF-16le", "UTF-16be"] as const)( + "orders a %s WITHOUT ROWID table by its decoded primary key", + async (encoding) => { + const path = await createDatabase( + `PRAGMA encoding='${encoding}'; CREATE TABLE keyed(k INTEGER, label TEXT, PRIMARY KEY(k)) WITHOUT ROWID; INSERT INTO keyed VALUES (2, 'b'), (1, 'a');`, + ); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "keyed", + row_limit: 20, + }); + expect(result.rows).toMatchObject({ order: "primary_key" }); + expect( + result.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([ + ["1", "a"], + ["2", "b"], + ]); + }, +); + +it("orders by a free rowid spelling when a virtual generated column shadows one", async () => { + const path = await createDatabase(` + CREATE TABLE generated(name TEXT, rowid TEXT GENERATED ALWAYS AS (name || '!') VIRTUAL); + INSERT INTO generated(name) VALUES ('b'), ('a');`); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "generated", + columns: ["name"], + row_limit: 20, + }); + // Rowid order is insertion order, which the generated expression reverses. + expect(result.rows).toMatchObject({ order: "rowid" }); + expect( + result.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([["b"], ["a"]]); +}); + +it("does not decode an unselected malformed primary-key name on a rowid table", async () => { + // `key` is the declared primary key, but a rowid table orders by an alias, so + // its name never reaches ORDER BY. A selection that omits it must stay + // readable even when the stored name is not valid UTF-8. + const path = await createApplicationDatabase( + `CREATE TABLE wide("key" INTEGER PRIMARY KEY, label TEXT); + INSERT INTO wide("key", label) VALUES (1, 'a'), (2, 'b');`, + (bytes) => { + const marker = Buffer.from('"key"'); + const offset = bytes.indexOf(marker); + if (offset < 0 || bytes.indexOf(marker, offset + 1) !== -1) + throw new Error("Expected unique authored key marker"); + bytes[offset + 1] = 0xff; + }, + ); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "wide", + schema_scope: "inventory", + columns: ["label"], + row_limit: 20, + }); + expect(result.rows).toMatchObject({ order: "rowid" }); + expect( + result.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([["a"], ["b"]]); +}); + +it("orders a WITHOUT ROWID table by its primary key, including DESC direction and collation", async () => { + const path = await createDatabase(` + CREATE TABLE ordered(k INTEGER, label TEXT COLLATE NOCASE, PRIMARY KEY(k DESC, label)) WITHOUT ROWID; + CREATE INDEX ordered_label ON ordered(label); + INSERT INTO ordered VALUES (1, 'b'), (3, 'a'), (2, 'c'), (3, 'd');`); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "ordered", + row_limit: 20, + }); + expect(result.rows).toMatchObject({ order: "primary_key", truncated: false }); + // k DESC, then label in its declared NOCASE order for equal keys. + expect( + result.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([ + ["3", "a"], + ["3", "d"], + ["2", "c"], + ["1", "b"], + ]); +}); + +it("keeps repeated primary-key terms in the order the index states them", async () => { + // PRIMARY KEY(a COLLATE BINARY, a COLLATE NOCASE DESC) is a valid composite + // key; collapsing the two terms to one column would advertise a unique order + // the key does not establish and read the offset window out of order. + const path = await createDatabase(` + CREATE TABLE repeated(a TEXT, PRIMARY KEY(a COLLATE BINARY, a COLLATE NOCASE DESC)) WITHOUT ROWID; + INSERT INTO repeated VALUES ('A'), ('a'), ('B');`); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "repeated", + row_limit: 20, + }); + expect(result.rows).toMatchObject({ order: "primary_key" }); + // BINARY order over the whole key: A (65), B (66), then a (97). Dropping the + // first term would sort by NOCASE DESC instead and put B first. + expect( + result.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([["A"], ["B"], ["a"]]); +}); + +it("orders by the first rowid spelling the table leaves unshadowed", async () => { + // Declaring `rowid` makes that spelling resolve to the stored column, so + // ORDER BY rowid would sort by a non-unique value instead of the row id. + const path = + await createDatabase(`CREATE TABLE shadowed(rowid TEXT, value TEXT); + INSERT INTO shadowed(rowid, value) VALUES ('z', 'first'), ('a', 'second'), ('z', 'third');`); + const first = inspectSqliteDatabaseSnapshot(path, { + path, + table: "shadowed", + row_limit: 2, + }); + expect(first.rows).toMatchObject({ + order: "rowid", + row_offset: 0, + truncated: true, + next_row_offset: 2, + }); + expect( + first.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([ + ["z", "first"], + ["a", "second"], + ]); + const second = inspectSqliteDatabaseSnapshot(path, { + path, + table: "shadowed", + row_limit: 2, + row_offset: first.rows?.next_row_offset ?? 0, + }); + // The page continues in insertion order rather than re-sorting by the text + // column that shadows `rowid`. + expect( + second.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([["z", "third"]]); +}); + +it("does not publish a next offset the row-offset rule would refuse", async () => { + const path = await createDatabase( + `CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT); + INSERT INTO shadowed VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`, + ); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "shadowed", + row_limit: 1, + }); + expect(result.rows).toMatchObject({ + order: "unspecified", + truncated: true, + next_row_offset: null, + }); +}); + +it("discovers a WITHOUT ROWID primary key the inventory projection omits", async () => { + const path = await createDatabase(` + CREATE TABLE keyed(k INTEGER, label TEXT, PRIMARY KEY(k)) WITHOUT ROWID; + INSERT INTO keyed VALUES (2, 'b'), (1, 'a');`); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "keyed", + schema_scope: "inventory", + columns: ["label"], + row_limit: 1, + }); + expect(result.rows).toMatchObject({ + order: "primary_key", + columns: ["label"], + truncated: true, + next_row_offset: 1, + }); + expect( + result.rows?.values.map((row) => + row.map((cell) => ("value" in cell ? cell.value : cell.type)), + ), + ).toEqual([["a"]]); +}); + +it("refuses a row offset when the table shadows rowid and declares no primary key", async () => { + const path = await createDatabase( + `CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT); + INSERT INTO shadowed VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`, + ); + expect( + inspectSqliteDatabaseSnapshot(path, { path, table: "shadowed" }).rows, + ).toMatchObject({ + order: "unspecified", + row_offset: 0, + next_row_offset: null, + }); + expect(() => + inspectSqliteDatabaseSnapshot(path, { + path, + table: "shadowed", + row_offset: 1, + }), + ).toThrow( + expect.objectContaining({ + // The refusal is about the caller's offset, not the selected table. + reason: "row-offset", + message: expect.stringContaining("row_offset"), + }), + ); +}); + nativeTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "rejects a runtime without native limits before querying the snapshot", async () => { diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index b3141a6219..fc19dd54c5 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -609,7 +609,9 @@ it("inspects committed WAL schema and rows while preserving every source artifac returned_rows: 1, truncated: true, row_limit: 1, - order: "unspecified", + order: "rowid", + row_offset: 0, + next_row_offset: 1, }, }); for (const artifact of original) { @@ -1212,6 +1214,72 @@ it.each(["UTF-8", "UTF-16le", "UTF-16be"] as const)( }, ); +it("returns the first sample of a truncated rowid-shadowing table without a next offset", async () => { + const workspace = await createTestWorkspace("rea-sqlite-shadowed-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("shadowed.db"); + const writer = new DatabaseSync(path); + writer.exec( + `CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT); + INSERT INTO shadowed VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`, + ); + writer.close(); + const provider = createProvider(); + onTestFinished(() => provider.close()); + const result = await provider.inspect({ + path, + table: "shadowed", + row_limit: 1, + }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + rows: { + table: "shadowed", + order: "unspecified", + row_limit: 1, + returned_rows: 1, + truncated: true, + row_offset: 0, + next_row_offset: null, + }, + }); +}); + +it("attributes a refused row offset to row_offset across the worker round trip", async () => { + const workspace = await createTestWorkspace("rea-sqlite-shadowed-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("shadowed.db"); + const writer = new DatabaseSync(path); + writer.exec( + `CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT); + INSERT INTO shadowed VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`, + ); + writer.close(); + const provider = createProvider(); + onTestFinished(() => provider.close()); + const result = await provider.inspect({ + path, + table: "shadowed", + row_offset: 1, + }); + if (result.ok) throw new Error("Expected a rejected row offset"); + // The worker refuses this offset itself, so the refusal has to survive the + // reply schema and arrive as an input error naming the caller's argument + // rather than as an unparsable reply. + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: ["row_offset"], + reason: "invalid_format", + message: expect.stringContaining("row_offset"), + }, + ], + }, + }); +}); + it("retains an unavailable virtual module definition without executing it", async () => { const workspace = await createTestWorkspace("rea-sqlite-virtual-"); onTestFinished(() => removeTestWorkspace(workspace.root));