fix(sqlite): page via declared rowid aliases (#11963)

Co-authored-by: hhang <169027046+xu91102@users.noreply.github.com>
This commit is contained in:
hhangandhhang authored and GitHub committed 2026-11-18 10:09:16 -05:00
1 parent 5fd543c59c
commit 090d5668c8
4 files changed
+127 -61

No files matched your search

+9 -5
View File
@@ -98,11 +98,15 @@ primary-key index's own direction and declared collation. `rows.row_offset`
reports the rows skipped before the window and `rows.next_row_offset` the offset
to request for the following page; it is `null` once the window reaches the end.
Pass `row_offset` (CLI `--row-offset`, requires `table`, default 0) to skip into
a table rather than repeating earlier rows. A table that declares all of
`rowid`, `_rowid_` and `oid` as columns and has no primary key reports
`order: "unspecified"` and refuses a non-zero `row_offset`, because the skipped
prefix would be arbitrary. Pages compose only while the reported database and
WAL SHA-256 match; re-inspecting a changed snapshot restarts the sequence.
a table rather than repeating earlier rows. If columns shadow all three
built-in names (`rowid`, `_rowid_`, and `oid`), REA can use a declared
[`INTEGER PRIMARY KEY` rowid alias](https://www.sqlite.org/lang_createtable.html#rowids_and_the_integer_primary_key),
even when that column is outside the requested projection. A separately
indexed key, including inline `INTEGER PRIMARY KEY DESC`, does not name the
rowid. With no usable order, the first sample reports `order: "unspecified"`
and a non-zero `row_offset` is refused. Pages compose only while the reported
database and WAL SHA-256 match; re-inspecting a changed snapshot restarts the
sequence.
View, virtual-table and shadow-table row reads are unsupported.
SQLite integers are decimal strings so every signed 64-bit value survives JSON
+1 -1
View File
@@ -22,7 +22,7 @@ export const SQLITE_TOOL_CONTRACTS = [
kind: "artifact-provider",
requiresActiveTarget: true,
description:
"Inspect a local SQLite snapshot and sibling WAL without modifying them. Returns their SHA-256 and schema inventory; select an exact ordinary table and columns for rows (default 20), or schema_scope database for expanded schema. row_offset pages a stated rowid/primary-key order only while database/WAL digests match; fully shadowed rowid aliases have unspecified order and cannot page. Values preserve 64-bit integers, text bytes and blobs with explicit truncation. Requires Node.js 24.x >=24.15 or 26+ with DatabaseSync.limits. worker_timeout_ms bounds worker startup/query/reply work, not snapshot acquisition, cleanup or host delivery; independent safety budgets remain. Supply a quiescent snapshot; no atomic live-snapshot claim. No arbitrary SQL, WHERE/ORDER BY, views/virtual tables, extensions, repair or encryption. Two inspections may run per provider; occupied capacity is retryable. Temporary-copy/worker bounds do not cap overall native heap.",
"Inspect a local SQLite snapshot and sibling WAL without modifying them. Returns their SHA-256 and schema inventory; select an exact ordinary table and columns for rows (default 20), or schema_scope database for expanded schema. row_offset pages a stated rowid/primary-key order while database/WAL digests match. A declared rowid alias can supply the order if all built-in aliases are shadowed. Values preserve 64-bit integers, text bytes and blobs with explicit truncation. Requires Node.js 24.x >=24.15 or 26+ with DatabaseSync.limits. worker_timeout_ms bounds worker startup/query/reply work, not snapshot acquisition, cleanup or host delivery; independent safety budgets remain. Supply a quiescent snapshot; no atomic live-snapshot claim. No arbitrary SQL, WHERE/ORDER BY, views/virtual tables, extensions, repair or encryption. Two inspections may run per provider; occupied capacity is retryable. Temporary-copy/worker bounds do not cap overall native heap.",
inputSchema: inspectSqliteDatabaseInputSchema,
outputSchema: evidenceResultOf(sqliteDatabaseSchema),
examples: [
+55 -29
View File
@@ -293,18 +293,15 @@ const readColumns = (
};
/**
* The facts a row window needs and the caller's projection cannot supply: which
* rowid alias spellings the table declares, and the primary key that orders a
* WITHOUT ROWID table. Alias detection compares raw name bytes against ASCII, so
* an unrelated malformed identifier is never decoded on the way.
* The table key a row window can order by independently of the caller's
* projection. Only a selected key needs to be decoded.
*/
type RowKeyFacts = {
readonly withoutRowid: boolean;
readonly primaryKey: readonly {
readonly name: string;
readonly position: number;
}[];
readonly shadowedAliases: ReadonlySet<string>;
readonly rowidAlias: string | undefined;
};
const ROWID_ALIASES = ["rowid", "_rowid_", "oid"] as const;
@@ -345,6 +342,40 @@ const asciiLower = (
return text;
};
const readPrimaryKeyIndexName = (
db: DatabaseSync,
table: string,
): Uint8Array | undefined => {
const index = db
.prepare(
"SELECT CAST(name AS BLOB) AS name FROM temp.pragma_index_list(?, 'main') WHERE origin = 'pk'",
)
.get(table);
return z
.object({ name: z.instanceof(Uint8Array) })
.optional()
.parse(index)?.name;
};
const readDeclaredRowidAlias = (
db: DatabaseSync,
table: string,
name: Uint8Array | undefined,
encoding: DatabaseEncoding,
): string | undefined => {
// On a rowid table, a declared primary key with no separate PK index
// names the rowid itself. Inline INTEGER PRIMARY KEY DESC has an index.
if (name === undefined || readPrimaryKeyIndexName(db, table) !== undefined)
return undefined;
try {
return decodeSchemaText(name, encoding, "rowid alias");
} catch (cause: unknown) {
// The caller can still read an unordered first sample without this key.
if (cause instanceof SqliteInspectionFailure) return undefined;
throw cause;
}
};
const readRowKeyFacts = (
db: DatabaseSync,
name: string,
@@ -370,6 +401,7 @@ const readRowKeyFacts = (
const withoutRowid = row !== undefined && row.wr !== 0 && row.wr !== 0n;
const shadowed = new Set<string>();
const primaryKey: { name: string; position: number }[] = [];
let declaredKey: Uint8Array | undefined;
for (const raw of db
.prepare(
"SELECT cid, CAST(name AS BLOB) AS name, pk FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 2, 3) ORDER BY cid",
@@ -391,22 +423,26 @@ const readRowKeyFacts = (
(ROWID_ALIASES as readonly string[]).includes(alias)
)
shadowed.add(alias);
// A rowid table's ORDER BY names an alias, never a declared key, so only a
// WITHOUT ROWID table's key columns are worth decoding — and a malformed
// unselected key name must not abort an otherwise readable selection.
const pk = Number(value.pk);
if (withoutRowid && pk > 0)
if (pk <= 0) continue;
if (withoutRowid) {
primaryKey.push({
name: decodeSchemaText(value.name, encoding, "primary key column"),
position: pk,
});
} else if (pk === 1) {
declaredKey = value.name;
}
}
const rowidAlias = withoutRowid
? undefined
: (ROWID_ALIASES.find((spelling) => !shadowed.has(spelling)) ??
readDeclaredRowidAlias(db, name, declaredKey, encoding));
return {
withoutRowid,
primaryKey: primaryKey.sort(
(left, right) => left.position - right.position,
),
shadowedAliases: shadowed,
rowidAlias,
};
};
@@ -866,9 +902,8 @@ const indexKeyRow = z.object({
/**
* Choose the unique order a window is read in, without accepting caller SQL.
* A rowid table exposes an implicit rowid unless all three spellings are
* declared columns. A WITHOUT ROWID table orders by its primary key, reusing the
* index's own direction and collation so no sorter is introduced.
* A rowid table can use an unshadowed built-in name or a declared rowid alias.
* A WITHOUT ROWID table reuses its key index's direction and collation.
*/
const readRowOrder = (
db: DatabaseSync,
@@ -879,23 +914,14 @@ const readRowOrder = (
// A non-ordinary table's keys are not probed, and none of its rows are read.
if (facts === undefined) return { order: "unspecified", clause: "" };
if (!table.without_rowid) {
// ORDER BY must name an alias SQLite still binds: a declared `rowid` column
// shadows that spelling and would sort by a stored column that need not be
// unique. Pick the first spelling the table leaves free.
const alias = ROWID_ALIASES.find(
(spelling) => !facts.shadowedAliases.has(spelling),
);
const alias = facts.rowidAlias;
return alias === undefined
? { order: "unspecified", clause: "" }
: { order: "rowid", clause: ` ORDER BY ${alias}` };
: { order: "rowid", clause: ` ORDER BY ${quoteIdentifier(alias)}` };
}
const key = facts.primaryKey;
if (key.length === 0) return { order: "unspecified", clause: "" };
const primaryKey = db
.prepare(
"SELECT CAST(name AS BLOB) AS name FROM temp.pragma_index_list(?) WHERE origin = 'pk'",
)
.get(table.name) as { readonly name: unknown } | undefined;
const primaryKey = readPrimaryKeyIndexName(db, table.name);
if (primaryKey === undefined) return { order: "unspecified", clause: "" };
// A WITHOUT ROWID key may repeat a column at different collations or
// directions — `PRIMARY KEY(a COLLATE BINARY, a COLLATE NOCASE DESC)` is
@@ -911,7 +937,7 @@ const readRowOrder = (
.prepare(
'SELECT CAST(name AS BLOB) AS name, "desc", CAST(coll AS BLOB) AS coll FROM temp.pragma_index_xinfo(CAST(? AS BLOB)) WHERE key = 1 ORDER BY seqno',
)
.iterate(primaryKey.name as Uint8Array) as Iterable<unknown>) {
.iterate(primaryKey) as Iterable<unknown>) {
const { name, desc, coll } = indexKeyRow.parse(raw);
const column = decodeSchemaText(name, encoding, "index column");
const detail = {
@@ -979,7 +1005,7 @@ const readRows = (
if (offset > 0 && order === "unspecified")
throw new SqliteInspectionFailure(
"row-offset",
`row_offset requires a stated row order, but table ${input.table} shadows rowid, _rowid_ and oid and declares no primary key. Read the table without row_offset.`,
`row_offset requires a stated row order, but no usable row order was established for table ${input.table}. Use row_offset 0 for the first sample.`,
);
const window = `${source}${clause} LIMIT ? OFFSET ?`;
const rows: NonNullable<SqliteDatabase["rows"]> = {
@@ -907,6 +907,40 @@ it("does not publish a next offset the row-offset rule would refuse", async () =
});
});
it("pages by a declared rowid alias when all standard names are shadowed", async () => {
const path = await createDatabase(`
CREATE TABLE keyed("item id" INTEGER PRIMARY KEY, "RoWiD" TEXT,
"_rowid_" TEXT, "OID" TEXT, label TEXT);
INSERT INTO keyed("item id", label) VALUES (20, 'later'), (4, 'first');
`);
const input = {
path,
table: "keyed",
schema_scope: "inventory" as const,
columns: ["label"],
row_limit: 1,
};
const first = inspectSqliteDatabaseSnapshot(path, input);
expect(first.rows).toMatchObject({
order: "rowid",
columns: ["label"],
values: [[{ type: "text", value: "first" }]],
truncated: true,
next_row_offset: 1,
});
const next = inspectSqliteDatabaseSnapshot(path, {
...input,
row_offset: 1,
});
expect(next.rows).toMatchObject({
order: "rowid",
row_offset: 1,
values: [[{ type: "text", value: "later" }]],
truncated: false,
next_row_offset: null,
});
});
it("discovers a WITHOUT ROWID primary key the inventory projection omits", async () => {
const path = await createDatabase(`
CREATE TABLE keyed(k INTEGER, label TEXT, PRIMARY KEY(k)) WITHOUT ROWID;
@@ -931,32 +965,34 @@ it("discovers a WITHOUT ROWID primary key the inventory projection omits", async
).toEqual([["a"]]);
});
it("refuses a row offset when the table shadows rowid and declares no primary key", async () => {
const path = await createDatabase(
`CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT);
INSERT INTO shadowed VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`,
);
expect(
inspectSqliteDatabaseSnapshot(path, { path, table: "shadowed" }).rows,
).toMatchObject({
order: "unspecified",
row_offset: 0,
next_row_offset: null,
});
expect(() =>
inspectSqliteDatabaseSnapshot(path, {
path,
table: "shadowed",
row_offset: 1,
}),
).toThrow(
expect.objectContaining({
// The refusal is about the caller's offset, not the selected table.
reason: "row-offset",
message: expect.stringContaining("row_offset"),
}),
);
});
it.each(["", ", key INTEGER PRIMARY KEY DESC", ", key INT PRIMARY KEY"])(
"refuses a row offset for shadowed built-in names without a declared rowid alias (%s)",
async (keyDeclaration) => {
const path = await createDatabase(
`CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT${keyDeclaration});
INSERT INTO shadowed(rowid, _rowid_, oid) VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`,
);
expect(
inspectSqliteDatabaseSnapshot(path, { path, table: "shadowed" }).rows,
).toMatchObject({
order: "unspecified",
row_offset: 0,
next_row_offset: null,
});
expect(() =>
inspectSqliteDatabaseSnapshot(path, {
path,
table: "shadowed",
row_offset: 1,
}),
).toThrow(
expect.objectContaining({
reason: "row-offset",
message: expect.stringContaining("row_offset"),
}),
);
},
);
nativeTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
"rejects a runtime without native limits before querying the snapshot",