fix(sqlite): attribute generated-column evaluation failures to the selection (#5879)

* fix(sqlite): attribute generated-column evaluation failures to the selection

A selected virtual generated column whose expression needs a function the
read-only connection lacks (or raises an SQL error for a sampled row) was
classified by the worker's catch-all as a malformed database and projected as
an invalid_format issue on `path`.

Row reads now attribute SQLITE_ERROR from selected cells by probing each
selected virtual generated column over the sampled rows, and report a
typed capability_unavailable refusal naming the table, each failing column
with SQLite's message, and the remaining columns to select. Other causes,
including corruption, TOOBIG and NOMEM, keep their classification.

Fixes #5719

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(sqlite): recommend only probed-readable columns after evaluation failures

Record every per-column probe failure with its own reason, discover stored
columns outside an inventory-scope selection without decoding unrelated
malformed names, and present recovery as the structured columns selection
instead of shell-quoted command text built from database-supplied names.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

docs/sqlite-databases.md

@@ -57,7 +57,12 @@ triggers or arbitrary SQL.
 Rows are optional. `table` selects one exact ordinary-table name. `columns`
 (CLI repeated `--column`) selects exact visible column names in the requested
 order; omission selects all visible columns, including generated columns.
-Unselected cells are not evaluated or projected. Malformed column-name bytes
+Unselected cells are not evaluated or projected. A selected virtual generated
+column whose expression needs a function REA's connection lacks (extensions and
+application functions are not loaded), or raises an SQL error for a sampled
+row, is refused as `capability_unavailable`. The error names the table and
+each failing selected column with SQLite's message, and recommends only columns
+whose sampled values were read; stored and other generated cells stay readable. Malformed column-name bytes
 cause a format refusal when all columns are requested. In inventory scope, an explicit valid subset can
 still be sampled without decoding unrelated malformed column names.
 Column selection requires a

src/sqlite/SqliteDatabaseInspection.ts

@@ -10,9 +10,14 @@ import {
 } from "../domain/sqlite/sqliteDatabase.js";
 import {
   SQLITE_DATABASE_LIMITS,
+  SqliteColumnEvaluationFailure,
   SqliteInspectionFailure,
+  sqlitePrimaryResultCode,
 } from "./SqliteDatabaseLimits.js";
 
+/** SQLite's generic SQL error, raised while compiling or evaluating an expression. */
+const SQLITE_ERROR = 1;
+
 const natural = z
   .bigint()
   .nonnegative()
@@ -419,6 +424,107 @@ const readSchema = (
   return { schema, selectedTable };
 };
 
+/** Byte length of one cell's projected representation, without copying its value. */
+const cellByteLength = (name: string): string => {
+  const column = quoteIdentifier(name);
+  return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`;
+};
+
+/** Ordinary and stored generated columns, read without evaluating an expression; skips undecodable names. */
+const storedColumnNames = (
+  db: DatabaseSync,
+  table: string,
+  encoding: DatabaseEncoding,
+): string[] => {
+  const names: string[] = [];
+  for (const raw of db
+    .prepare(
+      "SELECT CAST(name AS BLOB) AS name FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 3) ORDER BY cid",
+    )
+    .iterate(table)) {
+    const { name } = z.object({ name: z.instanceof(Uint8Array) }).parse(raw);
+    try {
+      names.push(decodeSchemaText(name, encoding, "column name"));
+    } catch (cause: unknown) {
+      // An unrelated malformed name cannot be selected exactly, so it is not recommended.
+      if (!(cause instanceof SqliteInspectionFailure)) throw cause;
+    }
+  }
+  return names;
+};
+
+/**
+ * Attribute an SQL error from selected cells to the virtual generated columns
+ * that raise it on this connection. Each column is probed alone over the same
+ * sampled rows, so only columns whose probe succeeds are recommended; any
+ * other cause keeps its original classification.
+ */
+const columnEvaluationFailure = (
+  db: DatabaseSync,
+  cause: unknown,
+  selection: {
+    readonly table: SelectedTableFacts;
+    readonly columns: readonly string[];
+    readonly source: string;
+    readonly limit: number;
+    readonly encoding: DatabaseEncoding;
+  },
+): unknown => {
+  if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return cause;
+  const { table, columns, source, limit, encoding } = selection;
+  // The aggregate evaluates exactly the sampled cells, as the length pass does.
+  const probe = (name: string): { readonly failure: unknown } | undefined => {
+    try {
+      db.prepare(
+        `SELECT count(${cellByteLength(name)}) FROM (SELECT ${quoteIdentifier(name)} FROM ${source} LIMIT ?)`,
+      ).get(limit);
+      return undefined;
+    } catch (failure: unknown) {
+      return { failure };
+    }
+  };
+  const failures = new Map<string, unknown>();
+  for (const name of new Set(columns)) {
+    const failed = probe(name);
+    if (failed !== undefined) failures.set(name, failed.failure);
+  }
+  const generated = new Set(
+    table.columns
+      .filter((column) => column.hidden === 2)
+      .map((column) => column.name),
+  );
+  if (
+    ![...failures].some(
+      ([name, failure]) =>
+        generated.has(name) &&
+        sqlitePrimaryResultCode(failure) === SQLITE_ERROR,
+    )
+  )
+    return cause;
+  const remaining = columns.filter((name) => !failures.has(name));
+  const readableStoredColumns = (): string[] => {
+    try {
+      return storedColumnNames(db, table.name, encoding).filter(
+        (name) => !failures.has(name) && probe(name) === undefined,
+      );
+    } catch {
+      // Discovery only refines recovery; it must not replace the attributed failure.
+      return [];
+    }
+  };
+  return new SqliteColumnEvaluationFailure(
+    `SQLite cannot evaluate selected column(s) of table ${JSON.stringify(table.name)} on REA's read-only connection: ${[
+      ...failures,
+    ]
+      .map(
+        ([name, failure]) =>
+          `${JSON.stringify(name)}: ${failure instanceof Error ? failure.message : String(failure)}`,
+      )
+      .join("; ")}`,
+    remaining.length > 0 ? remaining : readableStoredColumns(),
+  );
+};
+
 const readRows = (
   db: DatabaseSync,
   input: InspectSqliteDatabaseInput,
@@ -460,19 +566,6 @@ const readRows = (
     })
     .join(", ");
   const source = `main.${quoteIdentifier(input.table)} NOT INDEXED`;
-  const lengths = db.prepare(
-    `SELECT ${columns
-      .map((name) => {
-        const column = quoteIdentifier(name);
-        return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`;
-      })
-      .join(", ")} FROM ${source} LIMIT ?`,
-  );
-  lengths.setReturnArrays(true);
-  for (const raw of lengths.iterate(limit))
-    budget.beforeProjection(cellByteLengths.parse(raw));
-  const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`);
-  statement.setReturnArrays(true);
   const rows: NonNullable<SqliteDatabase["rows"]> = {
     table: input.table,
     columns,
@@ -482,13 +575,33 @@ const readRows = (
     truncated: false,
     order: "unspecified",
   };
-  for (const raw of statement.iterate(limit)) {
-    const rawRow = arrayRow.parse(raw);
-    // Hex expansion already belongs to the retained-output budget, before decoding/copying cells.
-    budget.beforeDecode(rawRow);
-    const values = rawRow.map((value) => cell(value, encoding));
-    budget.retain(values);
-    rows.values.push(values);
+  // SQLite compiles and evaluates only selected cells, so an SQL error here is
+  // attributed to the selected generated expressions rather than the file.
+  try {
+    const lengths = db.prepare(
+      `SELECT ${columns.map(cellByteLength).join(", ")} FROM ${source} LIMIT ?`,
+    );
+    lengths.setReturnArrays(true);
+    for (const raw of lengths.iterate(limit))
+      budget.beforeProjection(cellByteLengths.parse(raw));
+    const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`);
+    statement.setReturnArrays(true);
+    for (const raw of statement.iterate(limit)) {
+      const rawRow = arrayRow.parse(raw);
+      // Hex expansion already belongs to the retained-output budget, before decoding/copying cells.
+      budget.beforeDecode(rawRow);
+      const values = rawRow.map((value) => cell(value, encoding));
+      budget.retain(values);
+      rows.values.push(values);
+    }
+  } catch (cause: unknown) {
+    throw columnEvaluationFailure(db, cause, {
+      table,
+      columns,
+      source,
+      limit,
+      encoding,
+    });
   }
   rows.returned_rows = rows.values.length;
   rows.truncated =

src/sqlite/SqliteDatabaseLimits.ts

@@ -12,6 +12,7 @@ export class SqliteInspectionFailure extends Error {
       | "format"
       | "selection"
       | "column-selection"
+      | "column-evaluation"
       | "unsupported"
       | "unavailable"
       | "output-limit"
@@ -21,3 +22,20 @@ export class SqliteInspectionFailure extends Error {
     super(message);
   }
 }
+/** Selected generated cells the private connection cannot compute; the database stays readable. */
+export class SqliteColumnEvaluationFailure extends SqliteInspectionFailure {
+  constructor(
+    message: string,
+    /** Selected or stored columns that remain readable without the failed expressions. */
+    readonly readableColumns: readonly string[],
+  ) {
+    super("column-evaluation", message);
+  }
+}
+/** SQLite's primary result code for a native node:sqlite failure, if it carries one. */
+export const sqlitePrimaryResultCode = (cause: unknown): number | undefined =>
+  cause instanceof Error &&
+  "errcode" in cause &&
+  typeof cause.errcode === "number"
+    ? cause.errcode & 0xff
+    : undefined;

src/sqlite/SqliteDatabaseWorker.ts

@@ -4,7 +4,9 @@ import { inspectSqliteDatabaseInputSchema } from "../domain/sqlite/sqliteDatabas
 import { inspectSqliteDatabaseSnapshot } from "./SqliteDatabaseInspection.js";
 import {
   SQLITE_DATABASE_LIMITS,
+  SqliteColumnEvaluationFailure,
   SqliteInspectionFailure,
+  sqlitePrimaryResultCode,
 } from "./SqliteDatabaseLimits.js";
 
 const requestSchema = z.strictObject({
@@ -29,20 +31,28 @@ const main = async (): Promise<void> => {
       ),
     };
   } catch (cause: unknown) {
-    const code =
-      cause instanceof Error && "errcode" in cause ? cause.errcode : undefined;
-    reply = {
-      ok: false,
-      reason:
-        cause instanceof SqliteInspectionFailure
-          ? cause.reason
-          : code === 7
-            ? "memory"
-            : code === 18
-              ? "output-limit"
-              : "format",
-      message: cause instanceof Error ? cause.message : String(cause),
-    };
+    const code = sqlitePrimaryResultCode(cause);
+    const message = cause instanceof Error ? cause.message : String(cause);
+    reply =
+      cause instanceof SqliteColumnEvaluationFailure
+        ? {
+            ok: false,
+            reason: cause.reason,
+            message,
+            readable_columns: cause.readableColumns,
+          }
+        : {
+            ok: false,
+            reason:
+              cause instanceof SqliteInspectionFailure
+                ? cause.reason
+                : code === 7
+                  ? "memory"
+                  : code === 18
+                    ? "output-limit"
+                    : "format",
+            message,
+          };
   }
   let encoded = JSON.stringify(reply);
   if (Buffer.byteLength(encoded) > SQLITE_DATABASE_LIMITS.outputBytes)

src/sqlite/SqliteDatabaseWorkerExecution.ts

@@ -39,7 +39,7 @@ const {
   ...inspectionShape
 } = sqliteDatabaseSchema.shape;
 const inspectionSchema = z.strictObject(inspectionShape);
-const replySchema = z.discriminatedUnion("ok", [
+const replySchema = z.union([
   z.strictObject({ ok: z.literal(true), inspection: inspectionSchema }),
   z.strictObject({
     ok: z.literal(false),
@@ -54,6 +54,12 @@ const replySchema = z.discriminatedUnion("ok", [
     ]),
     message: z.string(),
   }),
+  z.strictObject({
+    ok: z.literal(false),
+    reason: z.literal("column-evaluation"),
+    message: z.string(),
+    readable_columns: z.array(z.string()),
+  }),
 ]);
 /** Replaceable launcher retains the shared owned-process supervision boundary. */
 export type SqliteWorkerLauncher = NonNullable<
@@ -80,6 +86,23 @@ const rejectedReply = (
       reply.message,
       { capturedOutput, userMessage: reply.message },
     );
+  if (reply.reason === "column-evaluation") {
+    const readable = reply.readable_columns;
+    return new AnalysisCapabilityUnavailableError(
+      "sqlite",
+      operation,
+      reply.message,
+      {
+        capturedOutput,
+        userMessage: reply.message,
+        remediationAction: `The database is readable; only the listed column values failed. REA's SQLite connection loads no extensions or application functions and disables schema trust. ${
+          readable.length === 0
+            ? "No other readable column was found"
+            : `Select columns whose values were read: columns: ${JSON.stringify(readable)} (CLI: one --column per name)`
+        }; omit table and use schema_scope: database to read the stored declarations without evaluating rows. Obtaining the failed values needs a SQLite connection able to evaluate them. Do not repeat the unchanged request.`,
+      },
+    );
+  }
   if (reply.reason === "memory")
     return new AnalysisResourceConstraintError(
       operation,

tests/acceptance/sqliteDatabaseCli.test.ts

@@ -796,3 +796,98 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
     }
   },
 );
+
+cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
+  "attributes an unevaluable generated column to its selection, not the readable database path",
+  async ({ cli }) => {
+    const root = await createTestTempDirectory("rea-sqlite-generated-");
+    const path = join(root, "hashes.db");
+    const producer = new DatabaseSync(path);
+    try {
+      // The producing application supplies a function REA's connection lacks.
+      producer.function(
+        "application_digest",
+        { deterministic: true },
+        (value) => (typeof value === "string" ? `digest:${value}` : null),
+      );
+      producer.exec(`
+        CREATE TABLE t(a TEXT, b TEXT GENERATED ALWAYS AS (application_digest(a)) VIRTUAL);
+        INSERT INTO t(a) VALUES ('hello'), ('world');
+      `);
+    } finally {
+      producer.close();
+    }
+    const before = await readFile(path);
+    const { call } = await connectLocalToolsMcp();
+    const message = `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "b": unknown function: application_digest()`;
+    const expectedError = {
+      code: "capability_unavailable",
+      retryable: false,
+      message,
+      details: { provider_id: "sqlite", reason: message },
+      remediation: {
+        action: expect.stringContaining(
+          'Select columns whose values were read: columns: ["a"] (CLI: one --column per name)',
+        ),
+      },
+    };
+    for (const columns of [["b", "a"], undefined]) {
+      const result = await cli.run({
+        arguments: [
+          "inspect-sqlite-database",
+          path,
+          "--table",
+          "t",
+          ...(columns ?? []).flatMap((column) => ["--column", column]),
+          "--json",
+        ],
+        environment: { REA_LOG_LEVEL: "silent" },
+      });
+      expect(result.exitCode, result.stdout + result.stderr).toBe(1);
+      expect(result.json).toMatchObject(expectedError);
+      expect(result.stdout).not.toContain("invalid_format");
+      const response = await call("inspect_sqlite_database", {
+        path,
+        table: "t",
+        ...(columns === undefined ? {} : { columns }),
+      });
+      expect(parseMcpToolError(response).error).toMatchObject(expectedError);
+    }
+    const base = await cli.run({
+      arguments: [
+        "inspect-sqlite-database",
+        path,
+        "--table",
+        "t",
+        "--column",
+        "a",
+        "--json",
+      ],
+      environment: { REA_LOG_LEVEL: "silent" },
+    });
+    expect(base.exitCode, base.stdout + base.stderr).toBe(0);
+    expect(base.json).toMatchObject({
+      normalized_result: { rows: { columns: ["a"], returned_rows: 2 } },
+    });
+    const schema = await call("inspect_sqlite_database", {
+      path,
+      schema_scope: "database",
+    });
+    expect(schema.isError).not.toBe(true);
+    expect(schema.structuredContent).toMatchObject({
+      normalized_result: {
+        rows: null,
+        schema: {
+          tables: [
+            {
+              name: "t",
+              sql: expect.stringContaining("application_digest(a)"),
+              columns: [{ name: "a" }, { name: "b", hidden: 2 }],
+            },
+          ],
+        },
+      },
+    });
+    expect(await readFile(path)).toEqual(before);
+  },
+);

tests/boundary/sqlite/sqliteDatabaseInspection.test.ts

@@ -364,6 +364,179 @@ it("rejects the combined representation of individually small generated cells",
   }
 });
 
+it("attributes compile and row evaluation errors to each failing generated column", async () => {
+  const workspace = await createTestWorkspace("rea-sqlite-evaluation-");
+  onTestFinished(() => removeTestWorkspace(workspace.root));
+  const path = workspace.path("private.db");
+  // Writable schema plants an expression that fails only for the second
+  // stored row; virtual cells are not stored, so record layout is unchanged.
+  const producer = new DatabaseSync(path, { defensive: false });
+  try {
+    producer.function("application_digest", { deterministic: true }, (value) =>
+      String(value),
+    );
+    producer.exec(`
+      CREATE TABLE t(
+        id INTEGER,
+        payload TEXT,
+        digest TEXT GENERATED ALWAYS AS (application_digest(payload)) VIRTUAL,
+        stored_digest TEXT GENERATED ALWAYS AS (application_digest(payload)) STORED,
+        field GENERATED ALWAYS AS (payload) VIRTUAL,
+        upper_payload GENERATED ALWAYS AS (upper(payload)) VIRTUAL
+      );
+      INSERT INTO t(id, payload) VALUES (1, '{"k":1}'), (2, 'not json');
+      PRAGMA writable_schema = ON;
+      UPDATE sqlite_schema
+        SET sql = replace(sql, 'field GENERATED ALWAYS AS (payload)', 'field GENERATED ALWAYS AS (json_extract(payload, ''$.k''))')
+        WHERE name = 't';
+      PRAGMA writable_schema = OFF;
+    `);
+  } finally {
+    producer.close();
+  }
+  const failure = (input: { columns?: string[]; row_limit?: number }) => {
+    try {
+      inspectSqliteDatabaseSnapshot(path, { path, table: "t", ...input });
+    } catch (cause: unknown) {
+      return cause;
+    }
+    throw new Error("Expected a generated-column evaluation failure");
+  };
+  const prefix = `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: `;
+  expect(failure({})).toMatchObject({
+    reason: "column-evaluation",
+    message: `${prefix}"digest": unknown function: application_digest(); "field": malformed JSON`,
+    readableColumns: ["id", "payload", "stored_digest", "upper_payload"],
+  });
+  expect(failure({ columns: ["field", "id", "field"] })).toMatchObject({
+    message: `${prefix}"field": malformed JSON`,
+    readableColumns: ["id"],
+  });
+  // With no readable selection left, stored columns remain available without evaluation.
+  expect(failure({ columns: ["digest"] })).toMatchObject({
+    message: `${prefix}"digest": unknown function: application_digest()`,
+    readableColumns: ["id", "payload", "stored_digest"],
+  });
+  // Only sampled rows are evaluated, and stored or built-in generated cells are read.
+  expect(
+    inspectSqliteDatabaseSnapshot(path, {
+      path,
+      table: "t",
+      columns: ["field", "stored_digest", "upper_payload"],
+      row_limit: 1,
+    }).rows,
+  ).toMatchObject({
+    values: [
+      [
+        { type: "integer", value: "1" },
+        { type: "text", value: '{"k":1}' },
+        { type: "text", value: '{"K":1}' },
+      ],
+    ],
+    truncated: true,
+  });
+});
+
+const createApplicationDatabase = async (
+  sql: string,
+  edit?: (bytes: Buffer) => void,
+): Promise<string> => {
+  const workspace = await createTestWorkspace("rea-sqlite-evaluation-");
+  onTestFinished(() => removeTestWorkspace(workspace.root));
+  const path = workspace.path("private.db");
+  const producer = new DatabaseSync(path);
+  try {
+    // The producing application supplies a function REA's connection lacks.
+    producer.function("application_digest", { deterministic: true }, (value) =>
+      String(value),
+    );
+    producer.exec(sql);
+  } finally {
+    producer.close();
+  }
+  if (edit !== undefined) {
+    const bytes = await readFile(path);
+    edit(bytes);
+    await writeFile(path, bytes);
+  }
+  return path;
+};
+
+it("never recommends a selected column whose own probe fails for another reason", async () => {
+  const path = await createApplicationDatabase(`
+    CREATE TABLE t(
+      id INTEGER,
+      digest TEXT GENERATED ALWAYS AS (application_digest(id)) VIRTUAL,
+      payload BLOB GENERATED ALWAYS AS (zeroblob(72 * 1024 * 1024)) VIRTUAL
+    );
+    INSERT INTO t(id) VALUES (1);
+  `);
+  expect(() =>
+    inspectSqliteDatabaseSnapshot(path, {
+      path,
+      table: "t",
+      columns: ["payload", "digest", "id"],
+    }),
+  ).toThrow(
+    expect.objectContaining({
+      reason: "column-evaluation",
+      message: `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "payload": string or blob too big; "digest": unknown function: application_digest()`,
+      readableColumns: ["id"],
+    }),
+  );
+});
+
+it("recommends stored columns outside an inventory-scope selection without decoding malformed names", async () => {
+  const path = await createApplicationDatabase(
+    `
+    CREATE TABLE t(
+      id INTEGER,
+      "col" TEXT,
+      payload TEXT,
+      digest TEXT GENERATED ALWAYS AS (application_digest(payload)) VIRTUAL
+    );
+    INSERT INTO t(id, "col", payload) VALUES (1, 'kept', 'value');
+  `,
+    (bytes) => {
+      const marker = Buffer.from('"col"');
+      const offset = bytes.indexOf(marker);
+      if (offset < 0 || bytes.indexOf(marker, offset + 1) !== -1)
+        throw new Error("Expected unique authored column marker");
+      bytes[offset + 1] = 0xff;
+    },
+  );
+  expect(() =>
+    inspectSqliteDatabaseSnapshot(path, {
+      path,
+      table: "t",
+      schema_scope: "inventory",
+      columns: ["digest"],
+    }),
+  ).toThrow(
+    expect.objectContaining({
+      reason: "column-evaluation",
+      message: `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "digest": unknown function: application_digest()`,
+      readableColumns: ["id", "payload"],
+    }),
+  );
+  // The recommended stored columns are themselves selectable in the same scope.
+  expect(
+    inspectSqliteDatabaseSnapshot(path, {
+      path,
+      table: "t",
+      schema_scope: "inventory",
+      columns: ["id", "payload"],
+    }).rows,
+  ).toMatchObject({
+    values: [
+      [
+        { type: "integer", value: "1" },
+        { type: "text", value: "value" },
+      ],
+    ],
+  });
+});
+
 it("samples above 1000 rows and safe integer limits within the byte budget", async () => {
   const path = await createDatabase(`CREATE TABLE items(value INTEGER);
     WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers WHERE value < 1002)