mirror of
https://github.com/morluto/rea.git
synced 2026-11-18 20:53:00 +00:00
fix(sqlite): page via declared rowid aliases (#11963)
Co-authored-by: hhang <169027046+xu91102@users.noreply.github.com>
This commit is contained in:
1 parent
5fd543c59c
commit
090d5668c8
4 files changed
+127
-61
No files matched your search
@@ -98,11 +98,15 @@ primary-key index's own direction and declared collation. `rows.row_offset`
|
||||
reports the rows skipped before the window and `rows.next_row_offset` the offset
|
||||
to request for the following page; it is `null` once the window reaches the end.
|
||||
Pass `row_offset` (CLI `--row-offset`, requires `table`, default 0) to skip into
|
||||
a table rather than repeating earlier rows. A table that declares all of
|
||||
`rowid`, `_rowid_` and `oid` as columns and has no primary key reports
|
||||
`order: "unspecified"` and refuses a non-zero `row_offset`, because the skipped
|
||||
prefix would be arbitrary. Pages compose only while the reported database and
|
||||
WAL SHA-256 match; re-inspecting a changed snapshot restarts the sequence.
|
||||
a table rather than repeating earlier rows. If columns shadow all three
|
||||
built-in names (`rowid`, `_rowid_`, and `oid`), REA can use a declared
|
||||
[`INTEGER PRIMARY KEY` rowid alias](https://www.sqlite.org/lang_createtable.html#rowids_and_the_integer_primary_key),
|
||||
even when that column is outside the requested projection. A separately
|
||||
indexed key, including inline `INTEGER PRIMARY KEY DESC`, does not name the
|
||||
rowid. With no usable order, the first sample reports `order: "unspecified"`
|
||||
and a non-zero `row_offset` is refused. Pages compose only while the reported
|
||||
database and WAL SHA-256 match; re-inspecting a changed snapshot restarts the
|
||||
sequence.
|
||||
View, virtual-table and shadow-table row reads are unsupported.
|
||||
|
||||
SQLite integers are decimal strings so every signed 64-bit value survives JSON
|
||||
|
||||
@@ -22,7 +22,7 @@ export const SQLITE_TOOL_CONTRACTS = [
|
||||
kind: "artifact-provider",
|
||||
requiresActiveTarget: true,
|
||||
description:
|
||||
"Inspect a local SQLite snapshot and sibling WAL without modifying them. Returns their SHA-256 and schema inventory; select an exact ordinary table and columns for rows (default 20), or schema_scope database for expanded schema. row_offset pages a stated rowid/primary-key order only while database/WAL digests match; fully shadowed rowid aliases have unspecified order and cannot page. Values preserve 64-bit integers, text bytes and blobs with explicit truncation. Requires Node.js 24.x >=24.15 or 26+ with DatabaseSync.limits. worker_timeout_ms bounds worker startup/query/reply work, not snapshot acquisition, cleanup or host delivery; independent safety budgets remain. Supply a quiescent snapshot; no atomic live-snapshot claim. No arbitrary SQL, WHERE/ORDER BY, views/virtual tables, extensions, repair or encryption. Two inspections may run per provider; occupied capacity is retryable. Temporary-copy/worker bounds do not cap overall native heap.",
|
||||
"Inspect a local SQLite snapshot and sibling WAL without modifying them. Returns their SHA-256 and schema inventory; select an exact ordinary table and columns for rows (default 20), or schema_scope database for expanded schema. row_offset pages a stated rowid/primary-key order while database/WAL digests match. A declared rowid alias can supply the order if all built-in aliases are shadowed. Values preserve 64-bit integers, text bytes and blobs with explicit truncation. Requires Node.js 24.x >=24.15 or 26+ with DatabaseSync.limits. worker_timeout_ms bounds worker startup/query/reply work, not snapshot acquisition, cleanup or host delivery; independent safety budgets remain. Supply a quiescent snapshot; no atomic live-snapshot claim. No arbitrary SQL, WHERE/ORDER BY, views/virtual tables, extensions, repair or encryption. Two inspections may run per provider; occupied capacity is retryable. Temporary-copy/worker bounds do not cap overall native heap.",
|
||||
inputSchema: inspectSqliteDatabaseInputSchema,
|
||||
outputSchema: evidenceResultOf(sqliteDatabaseSchema),
|
||||
examples: [
|
||||
|
||||
@@ -293,18 +293,15 @@ const readColumns = (
|
||||
};
|
||||
|
||||
/**
|
||||
* The facts a row window needs and the caller's projection cannot supply: which
|
||||
* rowid alias spellings the table declares, and the primary key that orders a
|
||||
* WITHOUT ROWID table. Alias detection compares raw name bytes against ASCII, so
|
||||
* an unrelated malformed identifier is never decoded on the way.
|
||||
* The table key a row window can order by independently of the caller's
|
||||
* projection. Only a selected key needs to be decoded.
|
||||
*/
|
||||
type RowKeyFacts = {
|
||||
readonly withoutRowid: boolean;
|
||||
readonly primaryKey: readonly {
|
||||
readonly name: string;
|
||||
readonly position: number;
|
||||
}[];
|
||||
readonly shadowedAliases: ReadonlySet<string>;
|
||||
readonly rowidAlias: string | undefined;
|
||||
};
|
||||
|
||||
const ROWID_ALIASES = ["rowid", "_rowid_", "oid"] as const;
|
||||
@@ -345,6 +342,40 @@ const asciiLower = (
|
||||
return text;
|
||||
};
|
||||
|
||||
const readPrimaryKeyIndexName = (
|
||||
db: DatabaseSync,
|
||||
table: string,
|
||||
): Uint8Array | undefined => {
|
||||
const index = db
|
||||
.prepare(
|
||||
"SELECT CAST(name AS BLOB) AS name FROM temp.pragma_index_list(?, 'main') WHERE origin = 'pk'",
|
||||
)
|
||||
.get(table);
|
||||
return z
|
||||
.object({ name: z.instanceof(Uint8Array) })
|
||||
.optional()
|
||||
.parse(index)?.name;
|
||||
};
|
||||
|
||||
const readDeclaredRowidAlias = (
|
||||
db: DatabaseSync,
|
||||
table: string,
|
||||
name: Uint8Array | undefined,
|
||||
encoding: DatabaseEncoding,
|
||||
): string | undefined => {
|
||||
// On a rowid table, a declared primary key with no separate PK index
|
||||
// names the rowid itself. Inline INTEGER PRIMARY KEY DESC has an index.
|
||||
if (name === undefined || readPrimaryKeyIndexName(db, table) !== undefined)
|
||||
return undefined;
|
||||
try {
|
||||
return decodeSchemaText(name, encoding, "rowid alias");
|
||||
} catch (cause: unknown) {
|
||||
// The caller can still read an unordered first sample without this key.
|
||||
if (cause instanceof SqliteInspectionFailure) return undefined;
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
const readRowKeyFacts = (
|
||||
db: DatabaseSync,
|
||||
name: string,
|
||||
@@ -370,6 +401,7 @@ const readRowKeyFacts = (
|
||||
const withoutRowid = row !== undefined && row.wr !== 0 && row.wr !== 0n;
|
||||
const shadowed = new Set<string>();
|
||||
const primaryKey: { name: string; position: number }[] = [];
|
||||
let declaredKey: Uint8Array | undefined;
|
||||
for (const raw of db
|
||||
.prepare(
|
||||
"SELECT cid, CAST(name AS BLOB) AS name, pk FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 2, 3) ORDER BY cid",
|
||||
@@ -391,22 +423,26 @@ const readRowKeyFacts = (
|
||||
(ROWID_ALIASES as readonly string[]).includes(alias)
|
||||
)
|
||||
shadowed.add(alias);
|
||||
// A rowid table's ORDER BY names an alias, never a declared key, so only a
|
||||
// WITHOUT ROWID table's key columns are worth decoding — and a malformed
|
||||
// unselected key name must not abort an otherwise readable selection.
|
||||
const pk = Number(value.pk);
|
||||
if (withoutRowid && pk > 0)
|
||||
if (pk <= 0) continue;
|
||||
if (withoutRowid) {
|
||||
primaryKey.push({
|
||||
name: decodeSchemaText(value.name, encoding, "primary key column"),
|
||||
position: pk,
|
||||
});
|
||||
} else if (pk === 1) {
|
||||
declaredKey = value.name;
|
||||
}
|
||||
}
|
||||
const rowidAlias = withoutRowid
|
||||
? undefined
|
||||
: (ROWID_ALIASES.find((spelling) => !shadowed.has(spelling)) ??
|
||||
readDeclaredRowidAlias(db, name, declaredKey, encoding));
|
||||
return {
|
||||
withoutRowid,
|
||||
primaryKey: primaryKey.sort(
|
||||
(left, right) => left.position - right.position,
|
||||
),
|
||||
shadowedAliases: shadowed,
|
||||
rowidAlias,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -866,9 +902,8 @@ const indexKeyRow = z.object({
|
||||
|
||||
/**
|
||||
* Choose the unique order a window is read in, without accepting caller SQL.
|
||||
* A rowid table exposes an implicit rowid unless all three spellings are
|
||||
* declared columns. A WITHOUT ROWID table orders by its primary key, reusing the
|
||||
* index's own direction and collation so no sorter is introduced.
|
||||
* A rowid table can use an unshadowed built-in name or a declared rowid alias.
|
||||
* A WITHOUT ROWID table reuses its key index's direction and collation.
|
||||
*/
|
||||
const readRowOrder = (
|
||||
db: DatabaseSync,
|
||||
@@ -879,23 +914,14 @@ const readRowOrder = (
|
||||
// A non-ordinary table's keys are not probed, and none of its rows are read.
|
||||
if (facts === undefined) return { order: "unspecified", clause: "" };
|
||||
if (!table.without_rowid) {
|
||||
// ORDER BY must name an alias SQLite still binds: a declared `rowid` column
|
||||
// shadows that spelling and would sort by a stored column that need not be
|
||||
// unique. Pick the first spelling the table leaves free.
|
||||
const alias = ROWID_ALIASES.find(
|
||||
(spelling) => !facts.shadowedAliases.has(spelling),
|
||||
);
|
||||
const alias = facts.rowidAlias;
|
||||
return alias === undefined
|
||||
? { order: "unspecified", clause: "" }
|
||||
: { order: "rowid", clause: ` ORDER BY ${alias}` };
|
||||
: { order: "rowid", clause: ` ORDER BY ${quoteIdentifier(alias)}` };
|
||||
}
|
||||
const key = facts.primaryKey;
|
||||
if (key.length === 0) return { order: "unspecified", clause: "" };
|
||||
const primaryKey = db
|
||||
.prepare(
|
||||
"SELECT CAST(name AS BLOB) AS name FROM temp.pragma_index_list(?) WHERE origin = 'pk'",
|
||||
)
|
||||
.get(table.name) as { readonly name: unknown } | undefined;
|
||||
const primaryKey = readPrimaryKeyIndexName(db, table.name);
|
||||
if (primaryKey === undefined) return { order: "unspecified", clause: "" };
|
||||
// A WITHOUT ROWID key may repeat a column at different collations or
|
||||
// directions — `PRIMARY KEY(a COLLATE BINARY, a COLLATE NOCASE DESC)` is
|
||||
@@ -911,7 +937,7 @@ const readRowOrder = (
|
||||
.prepare(
|
||||
'SELECT CAST(name AS BLOB) AS name, "desc", CAST(coll AS BLOB) AS coll FROM temp.pragma_index_xinfo(CAST(? AS BLOB)) WHERE key = 1 ORDER BY seqno',
|
||||
)
|
||||
.iterate(primaryKey.name as Uint8Array) as Iterable<unknown>) {
|
||||
.iterate(primaryKey) as Iterable<unknown>) {
|
||||
const { name, desc, coll } = indexKeyRow.parse(raw);
|
||||
const column = decodeSchemaText(name, encoding, "index column");
|
||||
const detail = {
|
||||
@@ -979,7 +1005,7 @@ const readRows = (
|
||||
if (offset > 0 && order === "unspecified")
|
||||
throw new SqliteInspectionFailure(
|
||||
"row-offset",
|
||||
`row_offset requires a stated row order, but table ${input.table} shadows rowid, _rowid_ and oid and declares no primary key. Read the table without row_offset.`,
|
||||
`row_offset requires a stated row order, but no usable row order was established for table ${input.table}. Use row_offset 0 for the first sample.`,
|
||||
);
|
||||
const window = `${source}${clause} LIMIT ? OFFSET ?`;
|
||||
const rows: NonNullable<SqliteDatabase["rows"]> = {
|
||||
|
||||
@@ -907,6 +907,40 @@ it("does not publish a next offset the row-offset rule would refuse", async () =
|
||||
});
|
||||
});
|
||||
|
||||
it("pages by a declared rowid alias when all standard names are shadowed", async () => {
|
||||
const path = await createDatabase(`
|
||||
CREATE TABLE keyed("item id" INTEGER PRIMARY KEY, "RoWiD" TEXT,
|
||||
"_rowid_" TEXT, "OID" TEXT, label TEXT);
|
||||
INSERT INTO keyed("item id", label) VALUES (20, 'later'), (4, 'first');
|
||||
`);
|
||||
const input = {
|
||||
path,
|
||||
table: "keyed",
|
||||
schema_scope: "inventory" as const,
|
||||
columns: ["label"],
|
||||
row_limit: 1,
|
||||
};
|
||||
const first = inspectSqliteDatabaseSnapshot(path, input);
|
||||
expect(first.rows).toMatchObject({
|
||||
order: "rowid",
|
||||
columns: ["label"],
|
||||
values: [[{ type: "text", value: "first" }]],
|
||||
truncated: true,
|
||||
next_row_offset: 1,
|
||||
});
|
||||
const next = inspectSqliteDatabaseSnapshot(path, {
|
||||
...input,
|
||||
row_offset: 1,
|
||||
});
|
||||
expect(next.rows).toMatchObject({
|
||||
order: "rowid",
|
||||
row_offset: 1,
|
||||
values: [[{ type: "text", value: "later" }]],
|
||||
truncated: false,
|
||||
next_row_offset: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("discovers a WITHOUT ROWID primary key the inventory projection omits", async () => {
|
||||
const path = await createDatabase(`
|
||||
CREATE TABLE keyed(k INTEGER, label TEXT, PRIMARY KEY(k)) WITHOUT ROWID;
|
||||
@@ -931,32 +965,34 @@ it("discovers a WITHOUT ROWID primary key the inventory projection omits", async
|
||||
).toEqual([["a"]]);
|
||||
});
|
||||
|
||||
it("refuses a row offset when the table shadows rowid and declares no primary key", async () => {
|
||||
const path = await createDatabase(
|
||||
`CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT);
|
||||
INSERT INTO shadowed VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`,
|
||||
);
|
||||
expect(
|
||||
inspectSqliteDatabaseSnapshot(path, { path, table: "shadowed" }).rows,
|
||||
).toMatchObject({
|
||||
order: "unspecified",
|
||||
row_offset: 0,
|
||||
next_row_offset: null,
|
||||
});
|
||||
expect(() =>
|
||||
inspectSqliteDatabaseSnapshot(path, {
|
||||
path,
|
||||
table: "shadowed",
|
||||
row_offset: 1,
|
||||
}),
|
||||
).toThrow(
|
||||
expect.objectContaining({
|
||||
// The refusal is about the caller's offset, not the selected table.
|
||||
reason: "row-offset",
|
||||
message: expect.stringContaining("row_offset"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
it.each(["", ", key INTEGER PRIMARY KEY DESC", ", key INT PRIMARY KEY"])(
|
||||
"refuses a row offset for shadowed built-in names without a declared rowid alias (%s)",
|
||||
async (keyDeclaration) => {
|
||||
const path = await createDatabase(
|
||||
`CREATE TABLE shadowed(rowid TEXT, _rowid_ TEXT, oid TEXT${keyDeclaration});
|
||||
INSERT INTO shadowed(rowid, _rowid_, oid) VALUES ('a', 'a', 'a'), ('b', 'b', 'b');`,
|
||||
);
|
||||
expect(
|
||||
inspectSqliteDatabaseSnapshot(path, { path, table: "shadowed" }).rows,
|
||||
).toMatchObject({
|
||||
order: "unspecified",
|
||||
row_offset: 0,
|
||||
next_row_offset: null,
|
||||
});
|
||||
expect(() =>
|
||||
inspectSqliteDatabaseSnapshot(path, {
|
||||
path,
|
||||
table: "shadowed",
|
||||
row_offset: 1,
|
||||
}),
|
||||
).toThrow(
|
||||
expect.objectContaining({
|
||||
reason: "row-offset",
|
||||
message: expect.stringContaining("row_offset"),
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
nativeTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
|
||||
"rejects a runtime without native limits before querying the snapshot",
|
||||
|
||||
Reference in new issue
Block a user