diff --git a/docs/guides/data/sqlite-databases.md b/docs/guides/data/sqlite-databases.md index a8bc798247..fc673a8b2f 100644 --- a/docs/guides/data/sqlite-databases.md +++ b/docs/guides/data/sqlite-databases.md @@ -98,11 +98,15 @@ primary-key index's own direction and declared collation. `rows.row_offset` reports the rows skipped before the window and `rows.next_row_offset` the offset to request for the following page; it is `null` once the window reaches the end. Pass `row_offset` (CLI `--row-offset`, requires `table`, default 0) to skip into -a table rather than repeating earlier rows. A table that declares all of -`rowid`, `_rowid_` and `oid` as columns and has no primary key reports -`order: "unspecified"` and refuses a non-zero `row_offset`, because the skipped -prefix would be arbitrary. Pages compose only while the reported database and -WAL SHA-256 match; re-inspecting a changed snapshot restarts the sequence. +a table rather than repeating earlier rows. If columns shadow all three +built-in names (`rowid`, `_rowid_`, and `oid`), REA can use a declared +[`INTEGER PRIMARY KEY` rowid alias](https://www.sqlite.org/lang_createtable.html#rowids_and_the_integer_primary_key), +even when that column is outside the requested projection. A separately +indexed key, including inline `INTEGER PRIMARY KEY DESC`, does not name the +rowid. With no usable order, the first sample reports `order: "unspecified"` +and a non-zero `row_offset` is refused. Pages compose only while the reported +database and WAL SHA-256 match; re-inspecting a changed snapshot restarts the +sequence. View, virtual-table and shadow-table row reads are unsupported. SQLite integers are decimal strings so every signed 64-bit value survives JSON diff --git a/src/contracts/sqlite/sqliteToolContracts.ts b/src/contracts/sqlite/sqliteToolContracts.ts index 46cc133246..bf91e54a1b 100644 --- a/src/contracts/sqlite/sqliteToolContracts.ts +++ b/src/contracts/sqlite/sqliteToolContracts.ts @@ -22,7 +22,7 @@ export const SQLITE_TOOL_CONTRACTS = [ kind: "artifact-provider", requiresActiveTarget: true, description: - "Inspect a local SQLite snapshot and sibling WAL without modifying them. Returns their SHA-256 and schema inventory; select an exact ordinary table and columns for rows (default 20), or schema_scope database for expanded schema. row_offset pages a stated rowid/primary-key order only while database/WAL digests match; fully shadowed rowid aliases have unspecified order and cannot page. Values preserve 64-bit integers, text bytes and blobs with explicit truncation. Requires Node.js 24.x >=24.15 or 26+ with DatabaseSync.limits. worker_timeout_ms bounds worker startup/query/reply work, not snapshot acquisition, cleanup or host delivery; independent safety budgets remain. Supply a quiescent snapshot; no atomic live-snapshot claim. No arbitrary SQL, WHERE/ORDER BY, views/virtual tables, extensions, repair or encryption. Two inspections may run per provider; occupied capacity is retryable. Temporary-copy/worker bounds do not cap overall native heap.", + "Inspect a local SQLite snapshot and sibling WAL without modifying them. Returns their SHA-256 and schema inventory; select an exact ordinary table and columns for rows (default 20), or schema_scope database for expanded schema. row_offset pages a stated rowid/primary-key order while database/WAL digests match. A declared rowid alias can supply the order if all built-in aliases are shadowed. Values preserve 64-bit integers, text bytes and blobs with explicit truncation. Requires Node.js 24.x >=24.15 or 26+ with DatabaseSync.limits. worker_timeout_ms bounds worker startup/query/reply work, not snapshot acquisition, cleanup or host delivery; independent safety budgets remain. Supply a quiescent snapshot; no atomic live-snapshot claim. No arbitrary SQL, WHERE/ORDER BY, views/virtual tables, extensions, repair or encryption. Two inspections may run per provider; occupied capacity is retryable. Temporary-copy/worker bounds do not cap overall native heap.", inputSchema: inspectSqliteDatabaseInputSchema, outputSchema: evidenceResultOf(sqliteDatabaseSchema), examples: [ diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index 7f31d16902..0384da516d 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -293,18 +293,15 @@ const readColumns = ( }; /** - * The facts a row window needs and the caller's projection cannot supply: which - * rowid alias spellings the table declares, and the primary key that orders a - * WITHOUT ROWID table. Alias detection compares raw name bytes against ASCII, so - * an unrelated malformed identifier is never decoded on the way. + * The table key a row window can order by independently of the caller's + * projection. Only a selected key needs to be decoded. */ type RowKeyFacts = { - readonly withoutRowid: boolean; readonly primaryKey: readonly { readonly name: string; readonly position: number; }[]; - readonly shadowedAliases: ReadonlySet; + readonly rowidAlias: string | undefined; }; const ROWID_ALIASES = ["rowid", "_rowid_", "oid"] as const; @@ -345,6 +342,40 @@ const asciiLower = ( return text; }; +const readPrimaryKeyIndexName = ( + db: DatabaseSync, + table: string, +): Uint8Array | undefined => { + const index = db + .prepare( + "SELECT CAST(name AS BLOB) AS name FROM temp.pragma_index_list(?, 'main') WHERE origin = 'pk'", + ) + .get(table); + return z + .object({ name: z.instanceof(Uint8Array) }) + .optional() + .parse(index)?.name; +}; + +const readDeclaredRowidAlias = ( + db: DatabaseSync, + table: string, + name: Uint8Array | undefined, + encoding: DatabaseEncoding, +): string | undefined => { + // On a rowid table, a declared primary key with no separate PK index + // names the rowid itself. Inline INTEGER PRIMARY KEY DESC has an index. + if (name === undefined || readPrimaryKeyIndexName(db, table) !== undefined) + return undefined; + try { + return decodeSchemaText(name, encoding, "rowid alias"); + } catch (cause: unknown) { + // The caller can still read an unordered first sample without this key. + if (cause instanceof SqliteInspectionFailure) return undefined; + throw cause; + } +}; + const readRowKeyFacts = ( db: DatabaseSync, name: string, @@ -370,6 +401,7 @@ const readRowKeyFacts = ( const withoutRowid = row !== undefined && row.wr !== 0 && row.wr !== 0n; const shadowed = new Set(); const primaryKey: { name: string; position: number }[] = []; + let declaredKey: Uint8Array | undefined; for (const raw of db .prepare( "SELECT cid, CAST(name AS BLOB) AS name, pk FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 2, 3) ORDER BY cid", @@ -391,22 +423,26 @@ const readRowKeyFacts = ( (ROWID_ALIASES as readonly string[]).includes(alias) ) shadowed.add(alias); - // A rowid table's ORDER BY names an alias, never a declared key, so only a - // WITHOUT ROWID table's key columns are worth decoding — and a malformed - // unselected key name must not abort an otherwise readable selection. const pk = Number(value.pk); - if (withoutRowid && pk > 0) + if (pk <= 0) continue; + if (withoutRowid) { primaryKey.push({ name: decodeSchemaText(value.name, encoding, "primary key column"), position: pk, }); + } else if (pk === 1) { + declaredKey = value.name; + } } + const rowidAlias = withoutRowid + ? undefined + : (ROWID_ALIASES.find((spelling) => !shadowed.has(spelling)) ?? + readDeclaredRowidAlias(db, name, declaredKey, encoding)); return { - withoutRowid, primaryKey: primaryKey.sort( (left, right) => left.position - right.position, ), - shadowedAliases: shadowed, + rowidAlias, }; }; @@ -866,9 +902,8 @@ const indexKeyRow = z.object({ /** * Choose the unique order a window is read in, without accepting caller SQL. - * A rowid table exposes an implicit rowid unless all three spellings are - * declared columns. A WITHOUT ROWID table orders by its primary key, reusing the - * index's own direction and collation so no sorter is introduced. + * A rowid table can use an unshadowed built-in name or a declared rowid alias. + * A WITHOUT ROWID table reuses its key index's direction and collation. */ const readRowOrder = ( db: DatabaseSync, @@ -879,23 +914,14 @@ const readRowOrder = ( // A non-ordinary table's keys are not probed, and none of its rows are read. if (facts === undefined) return { order: "unspecified", clause: "" }; if (!table.without_rowid) { - // ORDER BY must name an alias SQLite still binds: a declared `rowid` column - // shadows that spelling and would sort by a stored column that need not be - // unique. Pick the first spelling the table leaves free. - const alias = ROWID_ALIASES.find( - (spelling) => !facts.shadowedAliases.has(spelling), - ); + const alias = facts.rowidAlias; return alias === undefined ? { order: "unspecified", clause: "" } - : { order: "rowid", clause: ` ORDER BY ${alias}` }; + : { order: "rowid", clause: ` ORDER BY ${quoteIdentifier(alias)}` }; } const key = facts.primaryKey; if (key.length === 0) return { order: "unspecified", clause: "" }; - const primaryKey = db - .prepare( - "SELECT CAST(name AS BLOB) AS name FROM temp.pragma_index_list(?) WHERE origin = 'pk'", - ) - .get(table.name) as { readonly name: unknown } | undefined; + const primaryKey = readPrimaryKeyIndexName(db, table.name); if (primaryKey === undefined) return { order: "unspecified", clause: "" }; // A WITHOUT ROWID key may repeat a column at different collations or // directions — `PRIMARY KEY(a COLLATE BINARY, a COLLATE NOCASE DESC)` is @@ -911,7 +937,7 @@ const readRowOrder = ( .prepare( 'SELECT CAST(name AS BLOB) AS name, "desc", CAST(coll AS BLOB) AS coll FROM temp.pragma_index_xinfo(CAST(? AS BLOB)) WHERE key = 1 ORDER BY seqno', ) - .iterate(primaryKey.name as Uint8Array) as Iterable) { + .iterate(primaryKey) as Iterable) { const { name, desc, coll } = indexKeyRow.parse(raw); const column = decodeSchemaText(name, encoding, "index column"); const detail = { @@ -979,7 +1005,7 @@ const readRows = ( if (offset > 0 && order === "unspecified") throw new SqliteInspectionFailure( "row-offset", - `row_offset requires a stated row order, but table ${input.table} shadows rowid, _rowid_ and oid and declares no primary key. Read the table without row_offset.`, + `row_offset requires a stated row order, but no usable row order was established for table ${input.table}. Use row_offset 0 for the first sample.`, ); const window = `${source}${clause} LIMIT ? OFFSET ?`; const rows: NonNullable = { diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index aefc8e97c6..99eb8df494 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -907,6 +907,40 @@ it("does not publish a next offset the row-offset rule would refuse", async () = }); }); +it("pages by a declared rowid alias when all standard names are shadowed", async () => { + const path = await createDatabase(` + CREATE TABLE keyed("item id" INTEGER PRIMARY KEY, "RoWiD" TEXT, + "_rowid_" TEXT, "OID" TEXT, label TEXT); + INSERT INTO keyed("item id", label) VALUES (20, 'later'), (4, 'first'); + `); + const input = { + path, + table: "keyed", + schema_scope: "inventory" as const, + columns: ["label"], + row_limit: 1, + }; + const first = inspectSqliteDatabaseSnapshot(path, input); + expect(first.rows).toMatchObject({ + order: "rowid", + columns: ["label"], + values: [[{ type: "text", value: "first" }]], + truncated: true, + next_row_offset: 1, + }); + const next = inspectSqliteDatabaseSnapshot(path, { + ...input, + row_offset: 1, + }); + expect(next.rows).toMatchObject({ + order: "rowid", + row_offset: 1, + values: [[{ type: "text", value: "later" }]], + truncated: false, + next_row_offset: null, + }); +}); + it("discovers a WITHOUT ROWID primary key the inventory projection omits", async () => { const path = await createDatabase(` CREATE TABLE keyed(k INTEGER, label TEXT, PRIMARY KEY(k)) WITHOUT ROWID; @@ -931,32 +965,34 @@ it("discovers a WITHOUT ROWID primary key the inventory projection omits", async ).toEqual([["a"]]); }); -it("refuses a row offset when the table shadows rowid and declares no primary key", async () => { - const path = await createDatabase( - `CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT); - INSERT INTO shadowed VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`, - ); - expect( - inspectSqliteDatabaseSnapshot(path, { path, table: "shadowed" }).rows, - ).toMatchObject({ - order: "unspecified", - row_offset: 0, - next_row_offset: null, - }); - expect(() => - inspectSqliteDatabaseSnapshot(path, { - path, - table: "shadowed", - row_offset: 1, - }), - ).toThrow( - expect.objectContaining({ - // The refusal is about the caller's offset, not the selected table. - reason: "row-offset", - message: expect.stringContaining("row_offset"), - }), - ); -}); +it.each(["", ", key INTEGER PRIMARY KEY DESC", ", key INT PRIMARY KEY"])( + "refuses a row offset for shadowed built-in names without a declared rowid alias (%s)", + async (keyDeclaration) => { + const path = await createDatabase( + `CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT${keyDeclaration}); + INSERT INTO shadowed(rowid, _rowid_, oid) VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`, + ); + expect( + inspectSqliteDatabaseSnapshot(path, { path, table: "shadowed" }).rows, + ).toMatchObject({ + order: "unspecified", + row_offset: 0, + next_row_offset: null, + }); + expect(() => + inspectSqliteDatabaseSnapshot(path, { + path, + table: "shadowed", + row_offset: 1, + }), + ).toThrow( + expect.objectContaining({ + reason: "row-offset", + message: expect.stringContaining("row_offset"), + }), + ); + }, +); nativeTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "rejects a runtime without native limits before querying the snapshot",