mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
fix(sqlite): attribute generated-column evaluation failures to the selection (#5879)
* fix(sqlite): attribute generated-column evaluation failures to the selection A selected virtual generated column whose expression needs a function the read-only connection lacks (or raises an SQL error for a sampled row) was classified by the worker's catch-all as a malformed database and projected as an invalid_format issue on `path`. Row reads now attribute SQLITE_ERROR from selected cells by probing each selected virtual generated column over the sampled rows, and report a typed capability_unavailable refusal naming the table, each failing column with SQLite's message, and the remaining columns to select. Other causes, including corruption, TOOBIG and NOMEM, keep their classification. Fixes #5719 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(sqlite): recommend only probed-readable columns after evaluation failures Record every per-column probe failure with its own reason, discover stored columns outside an inventory-scope selection without decoding unrelated malformed names, and present recovery as the structured columns selection instead of shell-quoted command text built from database-supplied names. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
efab044f30
commit
bcfcd8b6e2
7 files changed
+473
-36
No files matched your search
@@ -10,9 +10,14 @@ import {
|
||||
} from "../domain/sqlite/sqliteDatabase.js";
|
||||
import {
|
||||
SQLITE_DATABASE_LIMITS,
|
||||
SqliteColumnEvaluationFailure,
|
||||
SqliteInspectionFailure,
|
||||
sqlitePrimaryResultCode,
|
||||
} from "./SqliteDatabaseLimits.js";
|
||||
|
||||
/** SQLite's generic SQL error, raised while compiling or evaluating an expression. */
|
||||
const SQLITE_ERROR = 1;
|
||||
|
||||
const natural = z
|
||||
.bigint()
|
||||
.nonnegative()
|
||||
@@ -419,6 +424,107 @@ const readSchema = (
|
||||
return { schema, selectedTable };
|
||||
};
|
||||
|
||||
/** Byte length of one cell's projected representation, without copying its value. */
|
||||
const cellByteLength = (name: string): string => {
|
||||
const column = quoteIdentifier(name);
|
||||
return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`;
|
||||
};
|
||||
|
||||
/** Ordinary and stored generated columns, read without evaluating an expression; skips undecodable names. */
|
||||
const storedColumnNames = (
|
||||
db: DatabaseSync,
|
||||
table: string,
|
||||
encoding: DatabaseEncoding,
|
||||
): string[] => {
|
||||
const names: string[] = [];
|
||||
for (const raw of db
|
||||
.prepare(
|
||||
"SELECT CAST(name AS BLOB) AS name FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 3) ORDER BY cid",
|
||||
)
|
||||
.iterate(table)) {
|
||||
const { name } = z.object({ name: z.instanceof(Uint8Array) }).parse(raw);
|
||||
try {
|
||||
names.push(decodeSchemaText(name, encoding, "column name"));
|
||||
} catch (cause: unknown) {
|
||||
// An unrelated malformed name cannot be selected exactly, so it is not recommended.
|
||||
if (!(cause instanceof SqliteInspectionFailure)) throw cause;
|
||||
}
|
||||
}
|
||||
return names;
|
||||
};
|
||||
|
||||
/**
|
||||
* Attribute an SQL error from selected cells to the virtual generated columns
|
||||
* that raise it on this connection. Each column is probed alone over the same
|
||||
* sampled rows, so only columns whose probe succeeds are recommended; any
|
||||
* other cause keeps its original classification.
|
||||
*/
|
||||
const columnEvaluationFailure = (
|
||||
db: DatabaseSync,
|
||||
cause: unknown,
|
||||
selection: {
|
||||
readonly table: SelectedTableFacts;
|
||||
readonly columns: readonly string[];
|
||||
readonly source: string;
|
||||
readonly limit: number;
|
||||
readonly encoding: DatabaseEncoding;
|
||||
},
|
||||
): unknown => {
|
||||
if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return cause;
|
||||
const { table, columns, source, limit, encoding } = selection;
|
||||
// The aggregate evaluates exactly the sampled cells, as the length pass does.
|
||||
const probe = (name: string): { readonly failure: unknown } | undefined => {
|
||||
try {
|
||||
db.prepare(
|
||||
`SELECT count(${cellByteLength(name)}) FROM (SELECT ${quoteIdentifier(name)} FROM ${source} LIMIT ?)`,
|
||||
).get(limit);
|
||||
return undefined;
|
||||
} catch (failure: unknown) {
|
||||
return { failure };
|
||||
}
|
||||
};
|
||||
const failures = new Map<string, unknown>();
|
||||
for (const name of new Set(columns)) {
|
||||
const failed = probe(name);
|
||||
if (failed !== undefined) failures.set(name, failed.failure);
|
||||
}
|
||||
const generated = new Set(
|
||||
table.columns
|
||||
.filter((column) => column.hidden === 2)
|
||||
.map((column) => column.name),
|
||||
);
|
||||
if (
|
||||
![...failures].some(
|
||||
([name, failure]) =>
|
||||
generated.has(name) &&
|
||||
sqlitePrimaryResultCode(failure) === SQLITE_ERROR,
|
||||
)
|
||||
)
|
||||
return cause;
|
||||
const remaining = columns.filter((name) => !failures.has(name));
|
||||
const readableStoredColumns = (): string[] => {
|
||||
try {
|
||||
return storedColumnNames(db, table.name, encoding).filter(
|
||||
(name) => !failures.has(name) && probe(name) === undefined,
|
||||
);
|
||||
} catch {
|
||||
// Discovery only refines recovery; it must not replace the attributed failure.
|
||||
return [];
|
||||
}
|
||||
};
|
||||
return new SqliteColumnEvaluationFailure(
|
||||
`SQLite cannot evaluate selected column(s) of table ${JSON.stringify(table.name)} on REA's read-only connection: ${[
|
||||
...failures,
|
||||
]
|
||||
.map(
|
||||
([name, failure]) =>
|
||||
`${JSON.stringify(name)}: ${failure instanceof Error ? failure.message : String(failure)}`,
|
||||
)
|
||||
.join("; ")}`,
|
||||
remaining.length > 0 ? remaining : readableStoredColumns(),
|
||||
);
|
||||
};
|
||||
|
||||
const readRows = (
|
||||
db: DatabaseSync,
|
||||
input: InspectSqliteDatabaseInput,
|
||||
@@ -460,19 +566,6 @@ const readRows = (
|
||||
})
|
||||
.join(", ");
|
||||
const source = `main.${quoteIdentifier(input.table)} NOT INDEXED`;
|
||||
const lengths = db.prepare(
|
||||
`SELECT ${columns
|
||||
.map((name) => {
|
||||
const column = quoteIdentifier(name);
|
||||
return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`;
|
||||
})
|
||||
.join(", ")} FROM ${source} LIMIT ?`,
|
||||
);
|
||||
lengths.setReturnArrays(true);
|
||||
for (const raw of lengths.iterate(limit))
|
||||
budget.beforeProjection(cellByteLengths.parse(raw));
|
||||
const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`);
|
||||
statement.setReturnArrays(true);
|
||||
const rows: NonNullable<SqliteDatabase["rows"]> = {
|
||||
table: input.table,
|
||||
columns,
|
||||
@@ -482,13 +575,33 @@ const readRows = (
|
||||
truncated: false,
|
||||
order: "unspecified",
|
||||
};
|
||||
for (const raw of statement.iterate(limit)) {
|
||||
const rawRow = arrayRow.parse(raw);
|
||||
// Hex expansion already belongs to the retained-output budget, before decoding/copying cells.
|
||||
budget.beforeDecode(rawRow);
|
||||
const values = rawRow.map((value) => cell(value, encoding));
|
||||
budget.retain(values);
|
||||
rows.values.push(values);
|
||||
// SQLite compiles and evaluates only selected cells, so an SQL error here is
|
||||
// attributed to the selected generated expressions rather than the file.
|
||||
try {
|
||||
const lengths = db.prepare(
|
||||
`SELECT ${columns.map(cellByteLength).join(", ")} FROM ${source} LIMIT ?`,
|
||||
);
|
||||
lengths.setReturnArrays(true);
|
||||
for (const raw of lengths.iterate(limit))
|
||||
budget.beforeProjection(cellByteLengths.parse(raw));
|
||||
const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`);
|
||||
statement.setReturnArrays(true);
|
||||
for (const raw of statement.iterate(limit)) {
|
||||
const rawRow = arrayRow.parse(raw);
|
||||
// Hex expansion already belongs to the retained-output budget, before decoding/copying cells.
|
||||
budget.beforeDecode(rawRow);
|
||||
const values = rawRow.map((value) => cell(value, encoding));
|
||||
budget.retain(values);
|
||||
rows.values.push(values);
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
throw columnEvaluationFailure(db, cause, {
|
||||
table,
|
||||
columns,
|
||||
source,
|
||||
limit,
|
||||
encoding,
|
||||
});
|
||||
}
|
||||
rows.returned_rows = rows.values.length;
|
||||
rows.truncated =
|
||||
|
||||
Reference in new issue
Block a user