fix(cli)!: refuse lossy text output (#7852)

* fix(cli)!: refuse lossy text output

* fix(cli): retain failure identity on text refusal

* fix(cli): check failed TOON token windows

* test(cli): bound keyed archive case duration

src/cli/cliOutputFormatter.ts

@@ -33,6 +33,58 @@ export interface CliDelimitedOutputOptions {
 
 type DelimitedScalar = string | number | boolean | null | undefined;
 
+/** UTF-8 cannot encode a lone UTF-16 surrogate without replacing it. */
+export const hasUnpairedSurrogate = (value: string): boolean =>
+  /[\ud800-\udfff]/u.test(value);
+
+/** Check selected output, including keys, before using a non-JSON text format. */
+export const containsUnpairedSurrogate = (value: unknown): boolean => {
+  return containsText(value, hasUnpairedSurrogate);
+};
+
+/** Markdown renderers replace NUL as well as UTF-8-invalid code units. */
+export const containsUnrepresentableMarkdown = (value: unknown): boolean =>
+  containsText(
+    value,
+    (text) => text.includes("\u0000") || hasUnpairedSurrogate(text),
+  );
+
+const containsText = (
+  value: unknown,
+  predicate: (text: string) => boolean,
+): boolean => {
+  const pending: Iterator<unknown>[] = [[value][Symbol.iterator]()];
+  const seen = new WeakSet<object>();
+  while (pending.length > 0) {
+    const next = pending.at(-1)!.next();
+    if (next.done === true) {
+      pending.pop();
+      continue;
+    }
+    const current: unknown = next.value;
+    if (typeof current === "string") {
+      if (predicate(current)) return true;
+      continue;
+    }
+    if (typeof current !== "object" || current === null || seen.has(current))
+      continue;
+    seen.add(current);
+    pending.push(outputValues(current));
+  }
+  return false;
+};
+
+function* outputValues(value: object): Generator<unknown> {
+  if (Array.isArray(value)) {
+    yield* value;
+    return;
+  }
+  for (const key of Object.keys(value)) {
+    yield key;
+    yield (value as Record<string, unknown>)[key];
+  }
+}
+
 export interface PreparedDelimitedOutput {
   readonly encodedHeaders: readonly string[];
   readonly rows: readonly (readonly DelimitedScalar[])[];
@@ -97,6 +149,14 @@ export const prepareDelimitedOutput = (
 
   const encodedHeaders: string[] = [];
   for (const field of fields) {
+    if (hasUnpairedSurrogate(field))
+      return err(
+        delimitedOutputInputError(
+          ["--columns"],
+          "CSV/TSV cannot preserve unpaired UTF-16 surrogates in UTF-8 output; use JSON.",
+          operation,
+        ),
+      );
     if (isSpreadsheetFormula(field))
       return err(
         delimitedOutputInputError(
@@ -135,6 +195,14 @@ export const prepareDelimitedOutput = (
             operation,
           ),
         );
+      if (typeof value === "string" && hasUnpairedSurrogate(value))
+        return err(
+          delimitedOutputInputError(
+            ["--columns", field],
+            "CSV/TSV cannot preserve unpaired UTF-16 surrogates in UTF-8 output; use JSON.",
+            operation,
+          ),
+        );
       preparedRow.push(value);
     }
     preparedRows.push(preparedRow);
@@ -250,6 +318,11 @@ const escapeInlineSyntax = (
 
 const escapeTableCell = (value: string): string =>
   value
+    .replace(
+      /\u0000|[\ud800-\udfff]/gu,
+      (character) =>
+        `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`,
+    )
     .replace(/&/g, "&amp;")
     .replace(/</g, "&lt;")
     .replace(/>/g, "&gt;")