diff --git a/src/application/desktop/DesktopBuildActionLog.ts b/src/application/desktop/DesktopBuildActionLog.ts index 6f530b2a7e..452e4e9a1e 100644 --- a/src/application/desktop/DesktopBuildActionLog.ts +++ b/src/application/desktop/DesktopBuildActionLog.ts @@ -1,6 +1,6 @@ import type { ComputerAction } from "../../domain/desktop/computerActions.js"; import type { JsonValue } from "../../domain/jsonValue.js"; -import { boundedUtf8Text } from "../../process/boundedUtf8Text.js"; +import { utf8HeadExcerpt } from "../../domain/utf8HeadExcerpt.js"; /** Bytes the log keeps of each string in an action's arguments, such as type's text. */ export const LOGGED_TEXT_BYTES = 1024; @@ -35,7 +35,7 @@ const bounded = ( redact: Redact, ): { readonly value: JsonValue; readonly cut: boolean } => { if (typeof value === "string") { - const { text, truncated } = boundedUtf8Text( + const { text, truncated } = utf8HeadExcerpt( redact(value), LOGGED_TEXT_BYTES, ); diff --git a/src/application/environment/environmentReplay.ts b/src/application/environment/environmentReplay.ts index 88f8d6ed1a..36bc105153 100644 --- a/src/application/environment/environmentReplay.ts +++ b/src/application/environment/environmentReplay.ts @@ -19,6 +19,7 @@ import { type EnvironmentSource, type RecordedExchange, } from "./environmentSource.js"; +import { utf8HeadExcerpt } from "../../domain/utf8HeadExcerpt.js"; /** Caller arguments as the MCP tool receives them. */ export type ReplayArguments = Readonly>; @@ -307,13 +308,6 @@ export const selectsRecordedLine = ( return true; }; -/** The longest prefix of text whose UTF-8 encoding fits the byte limit, encoding only that prefix. */ -const utf8Prefix = (text: string, maxBytes: number): string => { - // encodeInto stops before the first character that does not fit, so `read` ends on a boundary. - const { read } = new TextEncoder().encodeInto(text, new Uint8Array(maxBytes)); - return text.slice(0, read); -}; - /** Base64 characters, in whole groups, whose bytes fit the replay limit. */ const BASE64_LIMIT_CHARS = Math.floor(REPLAY_BODY_LIMIT_BYTES / 3) * 4; @@ -334,7 +328,7 @@ const replayBody = (exchange: RecordedExchange): ReplayBody => { : { body: body.encoded.slice(0, BASE64_LIMIT_CHARS), truncated: true }; case "cut": return { - body: utf8Prefix(body.text, REPLAY_BODY_LIMIT_BYTES), + body: utf8HeadExcerpt(body.text, REPLAY_BODY_LIMIT_BYTES).text, truncated: true, }; case "json": @@ -380,15 +374,18 @@ export const replayAnswer = ( exact: boolean, { body, truncated }: ReplayBody = replayBody(exchange), ): ReplayResult => { - const recordedPath = utf8Prefix(exchange.path, REPLAY_METADATA_LIMIT_BYTES); + const recordedPath = utf8HeadExcerpt( + exchange.path, + REPLAY_METADATA_LIMIT_BYTES, + ).text; const recordedQuery = exchange.query === null ? null - : utf8Prefix(exchange.query, REPLAY_METADATA_LIMIT_BYTES); + : utf8HeadExcerpt(exchange.query, REPLAY_METADATA_LIMIT_BYTES).text; const contentType = exchange.contentType === null ? null - : utf8Prefix(exchange.contentType, REPLAY_METADATA_LIMIT_BYTES); + : utf8HeadExcerpt(exchange.contentType, REPLAY_METADATA_LIMIT_BYTES).text; return { status: exchange.status, content_type: contentType, diff --git a/src/domain/artifactInspection.ts b/src/domain/artifactInspection.ts index 4a23822874..dc3ca9a661 100644 --- a/src/domain/artifactInspection.ts +++ b/src/domain/artifactInspection.ts @@ -25,8 +25,8 @@ import { evidenceSchema, parseEvidence, type Evidence } from "./evidence.js"; import { offsetText } from "./firmware/espImageStructures.js"; import { jsonValueSchema } from "./jsonValue.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; +import { nonEmptyTextSchema } from "./textSchemas.js"; -const textSchema = z.string().min(1); const evidenceIdSchema = prefixedDigestSchema("ev"); const artifactIdSchema = prefixedDigestSchema("art"); @@ -38,22 +38,22 @@ const inspectionObservationSchema = z.strictObject({ "container-header", "reconstruction", ]), - subject: textSchema, + subject: nonEmptyTextSchema, value: jsonValueSchema, evidence_id: evidenceIdSchema, }); const inspectionHypothesisSchema = z.strictObject({ hypothesis_id: prefixedDigestSchema("aih"), - statement: textSchema, + statement: nonEmptyTextSchema, confidence: z.enum(["medium", "low"]), basis_evidence_ids: z.array(evidenceIdSchema).min(1), - limitation: textSchema, + limitation: nonEmptyTextSchema, }); const inspectionContradictionBaseSchema = z.strictObject({ contradiction_id: prefixedDigestSchema("aic"), - statement: textSchema, + statement: nonEmptyTextSchema, occurrence_id: prefixedDigestSchema("occ"), evidence_id: evidenceIdSchema, @@ -66,8 +66,8 @@ const inspectionContradictionSchema = z.union([ ]); const nextProbeSchema = z.strictObject({ - operation: textSchema, - rationale: textSchema, + operation: nonEmptyTextSchema, + rationale: nonEmptyTextSchema, authority: z.enum([ "artifact-bytes", "managed-static-analysis", @@ -79,7 +79,7 @@ const nextProbeSchema = z.strictObject({ const unexploredBranchSchema = z.strictObject({ branch_id: prefixedDigestSchema("aib"), reason: z.enum(["format-specific-analysis-required", "unknown-format"]), - detail: textSchema, + detail: nonEmptyTextSchema, next_probe: nextProbeSchema.nullable(), evidence_id: evidenceIdSchema, }); @@ -91,7 +91,7 @@ export const artifactInspectionResultSchema = z.strictObject({ manifest_id: prefixedDigestSchema("agm"), root_artifact_id: artifactIdSchema, root_sha256: z.string().regex(/^[a-f0-9]{64}$/u), - root_format: textSchema, + root_format: nonEmptyTextSchema, }), substeps: z .array( @@ -125,7 +125,7 @@ export const artifactInspectionResultSchema = z.strictObject({ next_probes_omitted: z.int().min(0), }), evidence_links: z.array(evidenceIdSchema).min(1), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }); export type ArtifactInspectionResult = z.infer< diff --git a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts index 57a7a14c76..29f13c54ca 100644 --- a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts +++ b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts @@ -1,9 +1,8 @@ import { z } from "zod"; import { javascriptSourceRangeSchema } from "./javascriptSourceCoordinates.js"; import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; -const boundedTextSchema = z.string().min(1); -const boundedKeySchema = z.string().min(1); const unsafePathSegments = new Set(["", ".", ".."]); const isSafeRelativePath = (path: string): boolean => { if (path.startsWith("/")) return false; @@ -63,7 +62,7 @@ const applicationUnavailableReasonSchema = z.enum([ const unavailableFactSchema = z.strictObject({ available: z.literal(false), reason: applicationUnavailableReasonSchema, - detail: boundedTextSchema, + detail: nonEmptyTextSchema, }); const availableArtifactSchema = z @@ -89,7 +88,7 @@ const applicationArtifactReferenceSchema = z.discriminatedUnion("available", [ const sourceRangeSchema = javascriptSourceRangeSchema.safeExtend({ kind: z.literal("source-range"), - source: boundedTextSchema, + source: nonEmptyTextSchema, }); const fileOffsetRangeSchema = z @@ -122,19 +121,19 @@ const applicationLocationSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("runtime"), capture_sha256: digestSchema, - target_key: boundedKeySchema, - frame_key: boundedKeySchema.nullable(), - script_key: boundedKeySchema.nullable(), + target_key: nonEmptyTextSchema, + frame_key: nonEmptyTextSchema.nullable(), + script_key: nonEmptyTextSchema.nullable(), }), z.strictObject({ kind: z.literal("native-address"), - address: boundedKeySchema, - symbol: boundedTextSchema.nullable(), + address: nonEmptyTextSchema, + symbol: nonEmptyTextSchema.nullable(), }), z.strictObject({ kind: z.literal("package-export"), - package_name: boundedKeySchema, - export_path: boundedTextSchema, + package_name: nonEmptyTextSchema, + export_path: nonEmptyTextSchema, }), ]); @@ -247,7 +246,7 @@ const applicationGraphEvidenceBaseSchema = z.strictObject({ location: applicationGraphEvidenceLocationSchema, extractor: applicationExtractorSchema, coverage: applicationCoverageSchema, - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), evidence_ids: z.array(prefixedDigestSchema("ev")), }); @@ -439,7 +438,7 @@ export const applicationNodeIdentitySchema = z.discriminatedUnion("strategy", [ strategy: z.literal("artifact-local-key"), stability: z.literal("artifact-version"), artifact_sha256: digestSchema, - namespace: boundedKeySchema, + namespace: nonEmptyTextSchema, // The exact artifact-local value, such as `""` from `fetch("")`. key: z.string(), }), @@ -448,13 +447,13 @@ export const applicationNodeIdentitySchema = z.discriminatedUnion("strategy", [ stability: z.literal("capture-only"), capture_evidence_id: prefixedDigestSchema("ev"), capture_sha256: digestSchema, - runtime_key: boundedTextSchema, + runtime_key: nonEmptyTextSchema, }), z.strictObject({ strategy: z.literal("observation-fingerprint"), stability: z.literal("observation-only"), observation_sha256: digestSchema, - scope: boundedTextSchema, + scope: nonEmptyTextSchema, }), ]); diff --git a/src/domain/javascript/javascriptApplicationGraphSchemas.ts b/src/domain/javascript/javascriptApplicationGraphSchemas.ts index f97745a39b..f29062c8c1 100644 --- a/src/domain/javascript/javascriptApplicationGraphSchemas.ts +++ b/src/domain/javascript/javascriptApplicationGraphSchemas.ts @@ -8,8 +8,7 @@ import { } from "./javascriptApplicationEvidenceSchemas.js"; import { jsonValueSchema } from "../jsonValue.js"; import { prefixedDigestSchema } from "../digests.js"; - -const boundedTextSchema = z.string().min(1); +import { nonEmptyTextSchema } from "../textSchemas.js"; /** An observed IPC channel match whose uniqueness is not established. */ export const IPC_COVERAGE_LIMITED_PAIRING_BASIS = @@ -148,7 +147,7 @@ export const javascriptApplicationGraphInputSchema = z.strictObject({ nodes: z.array(applicationNodeSchema).min(1), edges: z.array(applicationEdgeSchema), coverage: applicationCoverageSchema, - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), }); /** Strict stored shape for a JavaScript Application Graph. */ diff --git a/src/domain/javascript/javascriptApplicationVersionComparisonSchemas.ts b/src/domain/javascript/javascriptApplicationVersionComparisonSchemas.ts index 772f2379cb..859ddaf286 100644 --- a/src/domain/javascript/javascriptApplicationVersionComparisonSchemas.ts +++ b/src/domain/javascript/javascriptApplicationVersionComparisonSchemas.ts @@ -8,10 +8,10 @@ import { } from "./javascriptApplicationGraph.js"; import { JAVASCRIPT_APPLICATION_NODE_KINDS } from "./javascriptApplicationGraphSchemas.js"; import { prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const nodeIdSchema = prefixedDigestSchema("jag_node"); -const textSchema = z.string().min(1); /** Check paired version identities and per-side native identity uniqueness. */ const validateApplicationVersionIdentities = ( @@ -128,7 +128,7 @@ const comparisonItemContextShape = { ]), ), evidence_links: z.array(evidenceIdSchema).min(2), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }; const comparisonItemSchema = z.union([ @@ -212,7 +212,7 @@ export const applicationVersionComparisonResultSchema = z graph: javascriptApplicationGraphSchema, coverage: comparisonCoverageSchema, evidence_links: z.array(evidenceIdSchema).min(2), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }) .superRefine((result, context) => { const sourceGraphsComplete = diff --git a/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts b/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts index b50690e703..028ee0540c 100644 --- a/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts +++ b/src/domain/javascript/javascriptExportShapeComparisonSchemas.ts @@ -4,11 +4,11 @@ import { javascriptSourceRangeSchema } from "./javascriptSourceCoordinates.js"; import { evidenceSchema } from "../evidence.js"; import { prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const graphIdSchema = prefixedDigestSchema("jag"); const nodeIdSchema = prefixedDigestSchema("jag_node"); -const textSchema = z.string().min(1); const exportNameSchema = z.string(); const semanticPrimitiveSchema = z.union([ z.string(), @@ -64,9 +64,9 @@ export const compareJavaScriptExportShapesInputSchema = z .strictObject({ left: evidenceSchema, right: evidenceSchema, - left_module_path: textSchema, + left_module_path: nonEmptyTextSchema, left_export_name: exportNameSchema, - right_module_path: textSchema, + right_module_path: nonEmptyTextSchema, right_export_name: exportNameSchema, }) .superRefine(validateJavaScriptExportShapeIdentities); @@ -92,7 +92,7 @@ const projectedReturnFieldSchema = z.discriminatedUnion("state", [ ...projectedReturnFieldShape, state: z.literal("unknown"), value: z.null(), - reason: textSchema, + reason: nonEmptyTextSchema, }), ]); export type ProjectedReturnField = z.infer; @@ -149,9 +149,9 @@ const projectedReturnShapeSchema = z export const projectedExportReturnShapesSchema = z .strictObject({ semantic_role: z.literal("export-return-shapes"), - module_path: textSchema, + module_path: nonEmptyTextSchema, exported_name: exportNameSchema, - callable_id: textSchema, + callable_id: nonEmptyTextSchema, callable_kind: z.enum(["function", "class", "method"]), static_return_shapes: z.array(projectedReturnShapeSchema), return_shape_coverage: z.strictObject({ @@ -177,7 +177,7 @@ export const projectedExportReturnShapesSchema = z const exportCandidateSchema = z.strictObject({ node_id: nodeIdSchema, - module_path: textSchema, + module_path: nonEmptyTextSchema, export_name: exportNameSchema, matches_requested_module: z.boolean(), matches_requested_export: z.boolean(), @@ -186,7 +186,7 @@ const exportCandidateSchema = z.strictObject({ const selectorResultShape = { evidence_id: evidenceIdSchema, graph_id: graphIdSchema, - requested_module_path: textSchema, + requested_module_path: nonEmptyTextSchema, requested_export_name: exportNameSchema, candidates: z.array(exportCandidateSchema), omitted_candidates: z.int().min(0), @@ -221,7 +221,7 @@ const valueAvailabilitySchema = z.discriminatedUnion("availability", [ }), z.strictObject({ availability: z.literal("unknown"), - reason: textSchema, + reason: nonEmptyTextSchema, }), ]); @@ -251,7 +251,7 @@ export const javaScriptExportShapeComparisonChangeSchema = z.strictObject({ left_source_range: javascriptSourceRangeSchema.nullable(), right_source_range: javascriptSourceRangeSchema.nullable(), evidence_links: z.array(evidenceIdSchema).length(2), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }); const inventoryPropertyCoverageSchema = z.strictObject({ @@ -311,7 +311,7 @@ export const javaScriptExportShapeComparisonResultSchema = z.strictObject({ omitted_changes: z.int().min(0), }), evidence_links: z.array(evidenceIdSchema).length(2), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }); export type ProjectedExportReturnShapes = z.output< diff --git a/src/domain/javascript/javascriptFeatureTraceSchemas.ts b/src/domain/javascript/javascriptFeatureTraceSchemas.ts index 46fcbcb73d..acc494c8b5 100644 --- a/src/domain/javascript/javascriptFeatureTraceSchemas.ts +++ b/src/domain/javascript/javascriptFeatureTraceSchemas.ts @@ -3,11 +3,11 @@ import { z } from "zod"; import { evidenceSchema, providerSchema } from "../evidence.js"; import { javascriptApplicationGraphSchema } from "./javascriptApplicationGraph.js"; import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const nodeIdSchema = prefixedDigestSchema("jag_node"); const edgeIdSchema = prefixedDigestSchema("jag_edge"); -const boundedTextSchema = z.string().min(1); /** Matching mode for a literal seed: string seeds default to containment. */ export const featureSeedMatchMode = (seed: { @@ -19,7 +19,7 @@ export const featureSeedMatchMode = (seed: { const apiSeedSchema = z.union([ z.strictObject({ kind: z.literal("api"), - value: boundedTextSchema, + value: nonEmptyTextSchema, match: z.enum(["exact", "contains"]).nullable().default(null), case_sensitive: z.boolean().default(false), }), @@ -34,7 +34,7 @@ const apiSeedSchema = z.union([ const apiPathSeedSchema = z.union([ z.strictObject({ kind: z.literal("api"), - value: boundedTextSchema, + value: nonEmptyTextSchema, match: z.enum(["exact", "contains"]).nullable().default(null), case_sensitive: z.boolean().default(false), entry_path: z.array(z.string()).min(1), @@ -63,7 +63,7 @@ export const applicationFeatureSeedSchema = z.union([ "module", "native-export", ]), - value: boundedTextSchema, + value: nonEmptyTextSchema, match: z.enum(["exact", "contains"]).nullable().default(null), case_sensitive: z.boolean().default(false), }), @@ -220,7 +220,7 @@ export const applicationFeatureTraceResultSchema = z.strictObject({ total_seed_matches: z.int().min(0), }), evidence_links: z.array(evidenceIdSchema).min(1), - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), }); export type TraceApplicationFeatureInput = z.infer< diff --git a/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts b/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts index 0d47a42b71..b678e67376 100644 --- a/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts +++ b/src/domain/javascript/javascriptRuntimeReconciliationSchemas.ts @@ -8,10 +8,10 @@ import { javascriptRuntimeUnresolvedLocationSchema } from "./javascriptRuntimeOb import { javascriptApplicationGraphSchema } from "./javascriptApplicationGraph.js"; import { digestSchema, prefixedDigestSchema } from "../digests.js"; import { urlDeclaresQueryOrFragment } from "../urlComponents.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const nodeIdSchema = prefixedDigestSchema("jag_node"); -const boundedTextSchema = z.string().min(1); const safePrefixSchema = z .string() .refine( @@ -171,7 +171,7 @@ const captureSummarySchema = z.strictObject({ capture_sha256: digestSchema, kind: z.enum(["browser", "electron", "v8-inspector", "electron-active"]), target_node_id: nodeIdSchema, - target_key: boundedTextSchema, + target_key: nonEmptyTextSchema, target_location: z .string() .min(1) @@ -282,7 +282,7 @@ export const javascriptRuntimeReconciliationResultSchema = z.strictObject({ used_for_primary_matching: z.literal(false), static_layer_count: z.int().min(0), runtime_script_declarations: z.int().min(0), - limitation: boundedTextSchema, + limitation: nonEmptyTextSchema, }), coverage: z.strictObject({ status: z.enum(["complete-within-inputs", "partial", "truncated"]), @@ -293,7 +293,7 @@ export const javascriptRuntimeReconciliationResultSchema = z.strictObject({ omitted_graph_items: z.int().min(0), }), evidence_links: z.array(evidenceIdSchema).min(2), - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), }); export type ReconcileJavaScriptRuntimeInput = z.infer< diff --git a/src/domain/javascript/javascriptSemanticGraphSchemas.ts b/src/domain/javascript/javascriptSemanticGraphSchemas.ts index 78abf10781..f7020859e9 100644 --- a/src/domain/javascript/javascriptSemanticGraphSchemas.ts +++ b/src/domain/javascript/javascriptSemanticGraphSchemas.ts @@ -9,8 +9,8 @@ import { } from "./javascriptApplicationEvidenceSchemas.js"; import { jsonValueSchema } from "../jsonValue.js"; import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; -const boundedTextSchema = z.string().min(1); const semanticNodeIdSchema = prefixedDigestSchema("jsrg_node"); const semanticRelationIdSchema = z .string() @@ -278,7 +278,7 @@ export const javaScriptSemanticUnknownInputSchema = z.strictObject({ family: z.enum(JAVASCRIPT_SEMANTIC_RELATION_FAMILIES), relation_kinds: z.array(z.enum(JAVASCRIPT_SEMANTIC_RELATIONS)).min(1), reason: z.enum(JAVASCRIPT_SEMANTIC_UNKNOWN_REASONS), - detail: boundedTextSchema, + detail: nonEmptyTextSchema, candidate_node_ids: z.array(semanticNodeIdSchema), evidence: applicationGraphEvidenceSchema, }); @@ -315,7 +315,7 @@ export const javaScriptSemanticFingerprintInputSchema = z.strictObject({ algorithm: z.literal("rea.javascript-semantic-function"), status: z.enum(["complete", "partial", "unavailable"]), components: fingerprintComponentsSchema, - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), evidence: applicationGraphEvidenceSchema, }); @@ -364,7 +364,7 @@ export const javaScriptSemanticGraphInputSchema = z.strictObject({ fingerprints: z.array(javaScriptSemanticFingerprintSchema), unknowns: z.array(javaScriptSemanticUnknownSchema), coverage: graphCoverageSchema, - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), }); /** Stored JavaScript Semantic Relation Graph. */ diff --git a/src/domain/javascript/sourceToBundleComparisonSchemas.ts b/src/domain/javascript/sourceToBundleComparisonSchemas.ts index d90918fead..3a82413d2d 100644 --- a/src/domain/javascript/sourceToBundleComparisonSchemas.ts +++ b/src/domain/javascript/sourceToBundleComparisonSchemas.ts @@ -5,10 +5,10 @@ import { evidenceSchema } from "../evidence.js"; import { JAVASCRIPT_APPLICATION_NODE_KINDS } from "./javascriptApplicationGraphSchemas.js"; import { historicalSourceGraphSchema } from "../referenceSourceGraph.js"; import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const nodeIdSchema = prefixedDigestSchema("jag_node"); -const textSchema = z.string().min(1); export const SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS = [ ["exact-source-digest", 100], @@ -39,8 +39,8 @@ export const compareSourceToBundleInputSchema = z.strictObject({ const sourceToBundleSignalSchema = z.strictObject({ kind: sourceToBundleSignalKindSchema, weight: z.int().min(1).max(100), - source_value: textSchema, - current_values: z.array(textSchema).min(1), + source_value: nonEmptyTextSchema, + current_values: z.array(nonEmptyTextSchema).min(1), }); const sourceToBundleCandidateSchema = z.strictObject({ @@ -55,9 +55,9 @@ const sourceToBundleCandidateSchema = z.strictObject({ const sourceToBundleItemSchema = z.strictObject({ mapping_id: prefixedDigestSchema("stbc_item"), - source_path: textSchema, + source_path: nonEmptyTextSchema, source_sha256: digestSchema.nullable(), - source_language: textSchema.nullable(), + source_language: nonEmptyTextSchema.nullable(), status: z.enum([ "unchanged", "modified", @@ -70,7 +70,7 @@ const sourceToBundleItemSchema = z.strictObject({ confidence: z.enum(["exact", "high", "medium", "unknown"]), current_node_ids: z.array(nodeIdSchema), candidates: z.array(sourceToBundleCandidateSchema), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }); /** Deterministic, evidence-bearing historical-source to shipped-bundle comparison. */ @@ -120,7 +120,7 @@ export const sourceToBundleComparisonResultSchema = z.strictObject({ candidate_evaluations: z.int().min(0), }), evidence_links: z.array(evidenceIdSchema).length(1), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }); export type SourceToBundleComparisonResult = z.infer< diff --git a/src/domain/managed/managedApplicationGraph.ts b/src/domain/managed/managedApplicationGraph.ts index dc54b7e056..e0fbd1def3 100644 --- a/src/domain/managed/managedApplicationGraph.ts +++ b/src/domain/managed/managedApplicationGraph.ts @@ -42,9 +42,9 @@ import { AnalysisInputError } from "../analysisErrorCore.js"; import type { EvidenceIntegrityError } from "../evidenceErrors.js"; import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { err, ok, type Result } from "../result.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); -const textSchema = z.string().min(1); const managedApplicationGraphReferenceShape = z.strictObject({ managed_artifact: evidenceInputSchema.optional(), @@ -135,7 +135,7 @@ export const managedApplicationGraphResultSchema = z status: z.enum(["complete-within-inputs", "partial"]), }), evidence_links: z.array(evidenceIdSchema).min(1).max(3), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }) .superRefine((result, context) => { const [rootId] = result.graph.root_node_ids; diff --git a/src/domain/managed/managedMemberComparison.ts b/src/domain/managed/managedMemberComparison.ts index 2023b523a0..b16f9753e7 100644 --- a/src/domain/managed/managedMemberComparison.ts +++ b/src/domain/managed/managedMemberComparison.ts @@ -25,9 +25,9 @@ import { import { keyMembers } from "./managedMemberComparisonMatch.js"; import { hasSignatureResolution } from "./managedMemberSignatureIdentity.js"; import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); -const boundedTextSchema = z.string().min(1); const memberComparisonShape = (source: Source) => ({ left: source, @@ -133,7 +133,7 @@ const comparisonItemSchema = < const comparisonItemContextShape = { evidence_links: z.array(evidenceIdSchema).length(2), - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), }; const methodComparisonItemSchema = comparisonItemSchema( @@ -221,7 +221,7 @@ export const managedMemberComparisonResultSchema = z.strictObject({ right_status: z.enum(["complete", "partial", "unavailable"]), }), evidence_links: z.array(evidenceIdSchema).length(2), - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), }); export type ManagedMemberComparisonResult = z.infer< diff --git a/src/domain/managed/managedNativeVerificationSchemas.ts b/src/domain/managed/managedNativeVerificationSchemas.ts index b322552aad..0ea1489dff 100644 --- a/src/domain/managed/managedNativeVerificationSchemas.ts +++ b/src/domain/managed/managedNativeVerificationSchemas.ts @@ -7,9 +7,9 @@ import { managedNativeImportSchema, } from "./managedArtifact.js"; import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); -const textSchema = z.string().min(1); const nativeSymbolSchema = z.strictObject({ evidence_id: evidenceIdSchema, @@ -43,7 +43,7 @@ const pinvokeVerificationContextShape = { item_id: prefixedDigestSchema("mnv_pinvoke"), managed: verifiedManagedDeclarationSchema, evidence_links: z.array(evidenceIdSchema).min(1), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }; const observedCandidatesSchema = z @@ -182,11 +182,11 @@ export const managedNativeVerificationResultSchema = z native_implementations: z.strictObject({ runtime_provided: z.int().nonnegative().nullable().default(null), unresolved: z.int().min(0), - reason: textSchema, + reason: nonEmptyTextSchema, }), coverage: verificationCoverageSchema, evidence_links: z.array(evidenceIdSchema).min(2), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }) .superRefine((result, context) => { const observedNativeCount = diff --git a/src/domain/managed/managedReconstruction.ts b/src/domain/managed/managedReconstruction.ts index 994c17dc4e..5b31d79952 100644 --- a/src/domain/managed/managedReconstruction.ts +++ b/src/domain/managed/managedReconstruction.ts @@ -17,13 +17,13 @@ import { readManagedInspectionEvidence } from "./managedInspectionEvidence.js"; import { AnalysisInputError } from "../analysisErrorCore.js"; import { EvidenceIntegrityError } from "../evidenceErrors.js"; import { err, ok, type Result } from "../result.js"; +import { nonEmptyTextSchema } from "../textSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); -const textSchema = z.string().min(1); const decompilerSchema = z.strictObject({ - name: textSchema, - version: textSchema.nullable().default(null), + name: nonEmptyTextSchema, + version: nonEmptyTextSchema.nullable().default(null), family: z.enum(["ilspy", "dnspy", "dnlib", "mono-cecil", "other"]), executable_sha256: digestSchema.nullable().default(null), options: z.array(z.string()).default([]), @@ -36,9 +36,9 @@ const managedReconstructionMethodInputSchema = z.strictObject({ reconstruction: z.strictObject({ kind: z.enum(["decompiled-csharp", "decompiled-il", "semantic-pseudocode"]), language: z.enum(["csharp", "il", "pseudocode"]), - text: textSchema, + text: nonEmptyTextSchema, text_sha256: digestSchema.optional(), - source_path: textSchema.nullable().optional(), + source_path: nonEmptyTextSchema.nullable().optional(), start_line: z.int().min(1).nullable().optional(), end_line: z.int().min(1).nullable().optional(), }), @@ -51,7 +51,7 @@ type ManagedReconstructionMethodInput = z.infer< const managedReconstructionImportRequestShape = { decompiler: decompilerSchema, methods: z.array(managedReconstructionMethodInputSchema).min(1), - notes: z.array(textSchema).default([]), + notes: z.array(nonEmptyTextSchema).default([]), }; const reconstructionInput = (source: Source) => @@ -79,11 +79,11 @@ const importedMethodSchema = z.strictObject({ reconstruction: z.strictObject({ kind: z.enum(["decompiled-csharp", "decompiled-il", "semantic-pseudocode"]), language: z.enum(["csharp", "il", "pseudocode"]), - text: textSchema, + text: nonEmptyTextSchema, text_sha256: digestSchema, text_length: z.int().min(1), line_count: z.int().min(1), - source_path: textSchema.nullable(), + source_path: nonEmptyTextSchema.nullable(), start_line: z.int().min(1).nullable(), end_line: z.int().min(1).nullable(), }), @@ -119,9 +119,9 @@ export const managedReconstructionImportResultSchema = z.strictObject({ total_text_bytes: z.int().min(1), }), methods: z.array(importedMethodSchema).min(1), - notes: z.array(textSchema), + notes: z.array(nonEmptyTextSchema), evidence_links: z.array(evidenceIdSchema).length(1), - limitations: z.array(textSchema), + limitations: z.array(nonEmptyTextSchema), }); export type ManagedReconstructionImportInput = z.infer< diff --git a/src/domain/reconstructionCoverage.ts b/src/domain/reconstructionCoverage.ts index b9d9b19030..daef9f6b92 100644 --- a/src/domain/reconstructionCoverage.ts +++ b/src/domain/reconstructionCoverage.ts @@ -34,7 +34,7 @@ const stableIdSchema = z export const reconstructionBoundaryIdSchema = stableIdSchema.describe( "Exact identifier of a boundary declared in the reconstruction coverage.", ); -const boundedTextSchema = z.string().trim().min(1); +const trimmedTextSchema = z.string().trim().min(1); const authoritySchema = z.enum([ "observed", @@ -57,7 +57,7 @@ const surfaceSchema = z.strictObject({ family: z.string().trim().min(1), artifact_id: stableIdSchema, occurrence_id: z.string().trim().min(1).nullable(), - location: boundedTextSchema, + location: trimmedTextSchema, authority: authoritySchema, dependency_surface_ids: z.array(stableIdSchema), evidence_ids: z.array(evidenceIdSchema).min(1), @@ -65,7 +65,7 @@ const surfaceSchema = z.strictObject({ const implementationOwnerSchema = z.strictObject({ disposition: z.literal("implemented"), - owner_path: boundedTextSchema, + owner_path: trimmedTextSchema, owner_export: z.string().trim().min(1).nullable(), owner_sha256: digestSchema, path_state: z.enum(["present", "missing", "unknown"]), @@ -75,7 +75,7 @@ const implementationOwnerSchema = z.strictObject({ const externalOwnerSchema = z.strictObject({ disposition: z.enum(["external", "non-goal"]), - rationale: boundedTextSchema, + rationale: trimmedTextSchema, }); const ownerSchema = z.strictObject({ @@ -284,7 +284,7 @@ const closureReasonSchema = z.strictObject({ "package-proof-unknown", ]), subject_id: stableIdSchema, - detail: boundedTextSchema, + detail: trimmedTextSchema, }); export const reconstructionClosureResultSchema = z.strictObject({ @@ -300,7 +300,7 @@ export const reconstructionClosureResultSchema = z.strictObject({ z.strictObject({ operation: stableIdSchema, subject_id: stableIdSchema, - rationale: boundedTextSchema, + rationale: trimmedTextSchema, }), ), evidence_ids: z.array(evidenceIdSchema), diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts index 6a3495e147..6812c2078b 100644 --- a/src/domain/referenceSourceGraph.ts +++ b/src/domain/referenceSourceGraph.ts @@ -10,7 +10,7 @@ import { } from "./referenceSourcePathIdentity.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; import { digestSchema, prefixedDigestSchema } from "./digests.js"; -const boundedTextSchema = z.string().min(1); +import { nonEmptyTextSchema } from "./textSchemas.js"; const relativePathSchema = z .string() .min(1) @@ -38,9 +38,9 @@ const classificationSchema = z.enum([ const entryBaseShape = { path: relativePathSchema, classifications: z.array(classificationSchema).min(1), - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), // Host presentation is retained evidence, separate from relocatable semantics. - local_diagnostics: z.array(boundedTextSchema).exactOptional(), + local_diagnostics: z.array(nonEmptyTextSchema).exactOptional(), }; const sourceFileBaseShape = { @@ -129,7 +129,7 @@ const sourceRelationshipSchema = z.strictObject({ const parseFailureSchema = z.strictObject({ path: relativePathSchema, parser: z.string().min(1), - reason: boundedTextSchema, + reason: nonEmptyTextSchema, }); const exclusionSchema = z.discriminatedUnion("reason", [ @@ -144,7 +144,7 @@ const exclusionSchema = z.discriminatedUnion("reason", [ "default-ignored", "caller-excluded", ]), - pattern: boundedTextSchema, + pattern: nonEmptyTextSchema, }), z.strictObject({ path: relativePathSchema, @@ -196,7 +196,7 @@ const graphShape = { .min(1) .regex(/^[\w .:@\x2f+\x2d]+$/u), }), - limitations: z.array(boundedTextSchema), + limitations: z.array(nonEmptyTextSchema), }; const historicalSourceGraphStructureSchema = z diff --git a/src/domain/textSchemas.ts b/src/domain/textSchemas.ts new file mode 100644 index 0000000000..eb1838f2c0 --- /dev/null +++ b/src/domain/textSchemas.ts @@ -0,0 +1,7 @@ +import { z } from "zod"; + +/** + * A string with at least one UTF-16 code unit. It bounds nothing else: apply a + * real length limit where a format, protocol or resource requires one. + */ +export const nonEmptyTextSchema = z.string().min(1); diff --git a/src/domain/utf8HeadExcerpt.test.ts b/src/domain/utf8HeadExcerpt.test.ts new file mode 100644 index 0000000000..4fc837c6ff --- /dev/null +++ b/src/domain/utf8HeadExcerpt.test.ts @@ -0,0 +1,61 @@ +import { expect, it } from "vitest"; + +import { utf8HeadExcerpt } from "./utf8HeadExcerpt.js"; + +it("keeps text within the budget unchanged", () => { + expect(utf8HeadExcerpt("abc", 3)).toEqual({ + text: "abc", + bytes: 3, + truncated: false, + }); + expect(utf8HeadExcerpt("", 0)).toEqual({ + text: "", + bytes: 0, + truncated: false, + }); +}); + +it("keeps the start and never splits a character", () => { + // "é" is 2 bytes, "€" 3 and "😀" 4 (a surrogate pair). + expect(utf8HeadExcerpt("aé€z", 5)).toEqual({ + text: "aé", + bytes: 3, + truncated: true, + }); + expect(utf8HeadExcerpt("aé€z", 6)).toEqual({ + text: "aé€", + bytes: 6, + truncated: true, + }); + expect(utf8HeadExcerpt("😀x", 3)).toEqual({ + text: "", + bytes: 0, + truncated: true, + }); + expect(utf8HeadExcerpt("😀x", 4)).toEqual({ + text: "😀", + bytes: 4, + truncated: true, + }); +}); + +it("charges a lone surrogate as its 3-byte replacement and keeps the source unit", () => { + expect(utf8HeadExcerpt("\uD800ab", 4)).toEqual({ + text: "\uD800a", + bytes: 4, + truncated: true, + }); +}); + +it("returns nothing for a non-positive budget", () => { + expect(utf8HeadExcerpt("abc", 0)).toEqual({ + text: "", + bytes: 0, + truncated: true, + }); + expect(utf8HeadExcerpt("abc", -1)).toEqual({ + text: "", + bytes: 0, + truncated: true, + }); +}); diff --git a/src/domain/utf8HeadExcerpt.ts b/src/domain/utf8HeadExcerpt.ts new file mode 100644 index 0000000000..554c33ae2d --- /dev/null +++ b/src/domain/utf8HeadExcerpt.ts @@ -0,0 +1,28 @@ +/** + * The longest prefix of `text` whose UTF-8 encoding fits `maximumBytes`, + * never splitting a character. `text` is a slice of the source, `bytes` its + * UTF-8 length, and `truncated` says whether any of the end was dropped. Only + * the retained prefix is encoded. + */ +export const utf8HeadExcerpt = ( + text: string, + maximumBytes: number, +): { + readonly text: string; + readonly bytes: number; + readonly truncated: boolean; +} => { + if (maximumBytes <= 0) + return { text: "", bytes: 0, truncated: text.length > 0 }; + // A UTF-16 code unit encodes to at most three UTF-8 bytes, and encodeInto + // stops before the first character that does not fit. + const { read, written } = new TextEncoder().encodeInto( + text, + new Uint8Array(Math.min(maximumBytes, text.length * 3)), + ); + return { + text: text.slice(0, read), + bytes: written, + truncated: read < text.length, + }; +}; diff --git a/src/gdb/GdbSessionManager.ts b/src/gdb/GdbSessionManager.ts index 8c515f4979..cc01fe8e98 100644 --- a/src/gdb/GdbSessionManager.ts +++ b/src/gdb/GdbSessionManager.ts @@ -57,6 +57,7 @@ import { type ProcessStartTimeReader, verifyOwnedInferiorsEnded, } from "../process/ownedDebuggerSession.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; const STARTUP_TIMEOUT_MS = 10_000; @@ -194,15 +195,7 @@ const truncateUtf8RecordPrefix = (line: string): string => { const budget = MAX_RECENT_MI_LINE_BYTES - Buffer.byteLength(RECENT_RECORD_TRUNCATION_MARKER, "utf8"); - let prefix = ""; - let retainedBytes = 0; - for (const codePoint of line) { - const codePointBytes = Buffer.byteLength(codePoint, "utf8"); - if (retainedBytes + codePointBytes > budget) break; - prefix += codePoint; - retainedBytes += codePointBytes; - } - return `${prefix}${RECENT_RECORD_TRUNCATION_MARKER}`; + return `${utf8HeadExcerpt(line, budget).text}${RECENT_RECORD_TRUNCATION_MARKER}`; }; /** Bound parser memory when a capped MI record contains extreme nesting. */ diff --git a/src/ghidra/GhidraJavaProbe.ts b/src/ghidra/GhidraJavaProbe.ts index e28e68649b..80d58109ba 100644 --- a/src/ghidra/GhidraJavaProbe.ts +++ b/src/ghidra/GhidraJavaProbe.ts @@ -21,7 +21,7 @@ import { prepareOwnedCommand, } from "../process/OwnedCommand.js"; import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; -import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; import { constants, accessSync, statSync } from "node:fs"; import { posix, win32 } from "node:path"; import { err, ok, type Result } from "../domain/result.js"; @@ -87,11 +87,11 @@ const JAVA_DIAGNOSTIC_STREAM_BYTES = 8192; const capturedProbeOutput = ( observed: ProviderProcessSnapshot, ): AnalysisCapturedOutput => { - const stdout = boundedUtf8Text( + const stdout = utf8HeadExcerpt( observed.stdout.text, JAVA_DIAGNOSTIC_STREAM_BYTES, ); - const stderr = boundedUtf8Text( + const stderr = utf8HeadExcerpt( observed.stderr.text, JAVA_DIAGNOSTIC_STREAM_BYTES, ); diff --git a/src/ghidra/GhidraLauncher.ts b/src/ghidra/GhidraLauncher.ts index cbc2456018..ef3420d26e 100644 --- a/src/ghidra/GhidraLauncher.ts +++ b/src/ghidra/GhidraLauncher.ts @@ -41,7 +41,7 @@ import { execFileOutputFailure, execFileFailureCapturedOutput, } from "../process/ExecFileOutput.js"; -import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js"; import { cleanupOwnedProcessGroup, @@ -522,7 +522,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { session.token, "[REDACTED]", ); - const bounded = boundedUtf8Text( + const bounded = utf8HeadExcerpt( redacted, GHIDRA_LAUNCH_COMMAND_OUTPUT_BYTES, ); diff --git a/src/ghidra/GhidraSessionError.ts b/src/ghidra/GhidraSessionError.ts index ce18daf02c..c76184e22b 100644 --- a/src/ghidra/GhidraSessionError.ts +++ b/src/ghidra/GhidraSessionError.ts @@ -5,7 +5,7 @@ import { } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; -import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; import type { GhidraDiagnosticsScope } from "./GhidraDiagnostics.js"; import { isGhidraByteSearchCursorMismatch } from "./GhidraByteSearchCursorFailure.js"; import { ghidraFileOffsetOutcome } from "./GhidraFileOffsetFailure.js"; @@ -264,7 +264,7 @@ const describeCause = ( return describeErrorCause(cause, redact, depth); } if (typeof cause === "string") { - const message = boundedUtf8Text(redact(cause), MAX_CAUSE_MESSAGE_BYTES); + const message = utf8HeadExcerpt(redact(cause), MAX_CAUSE_MESSAGE_BYTES); return { type: "string", message: message.text, @@ -283,11 +283,11 @@ const describeErrorCause = ( redact: (value: string, incomplete?: boolean) => string, depth: number, ): JsonValue => { - const message = boundedUtf8Text( + const message = utf8HeadExcerpt( redact(cause.message), MAX_CAUSE_MESSAGE_BYTES, ); - const name = boundedUtf8Text(redact(cause.name), MAX_CAUSE_FIELD_BYTES); + const name = utf8HeadExcerpt(redact(cause.name), MAX_CAUSE_FIELD_BYTES); return { name: name.text, ...(name.truncated ? { name_truncated: true } : {}), @@ -311,11 +311,11 @@ const causeCommandOutcome = ( if (!(cause instanceof GhidraLaunchSupportError)) return {}; const outcome = cause.outcome; if (outcome === undefined) return {}; - const stdout = boundedUtf8Text( + const stdout = utf8HeadExcerpt( redact(outcome.stdout, outcome.outputTruncated !== false), MAX_COMMAND_OUTPUT_BYTES, ); - const stderr = boundedUtf8Text( + const stderr = utf8HeadExcerpt( redact(outcome.stderr, outcome.outputTruncated !== false), MAX_COMMAND_OUTPUT_BYTES, ); @@ -345,7 +345,7 @@ const causeAdmissionDetail = ( detail[key] = value; continue; } - const bounded = boundedUtf8Text(redact(value), MAX_CAUSE_FIELD_BYTES); + const bounded = utf8HeadExcerpt(redact(value), MAX_CAUSE_FIELD_BYTES); detail[key] = bounded.text; } return { detail }; @@ -357,7 +357,7 @@ const causeCode = ( ): Record => { const code = "code" in cause ? cause.code : undefined; if (typeof code === "string") { - const bounded = boundedUtf8Text(redact(code), MAX_CAUSE_FIELD_BYTES); + const bounded = utf8HeadExcerpt(redact(code), MAX_CAUSE_FIELD_BYTES); return { code: bounded.text, ...(bounded.truncated ? { code_truncated: true } : {}), @@ -373,7 +373,7 @@ const causeSignal = ( ): Record => { const signal = "signal" in cause ? cause.signal : undefined; if (typeof signal === "string") { - const bounded = boundedUtf8Text(redact(signal), MAX_CAUSE_FIELD_BYTES); + const bounded = utf8HeadExcerpt(redact(signal), MAX_CAUSE_FIELD_BYTES); return { signal: bounded.text, ...(bounded.truncated ? { signal_truncated: true } : {}), @@ -390,7 +390,7 @@ const causeCompilerOutcome = ( if (!(cause instanceof DarwinProcessOwnershipInspectionError)) return {}; const outcome = cause.compilerOutcome; if (outcome === undefined) return {}; - const stderr = boundedUtf8Text( + const stderr = utf8HeadExcerpt( redact(outcome.stderr), MAX_COMPILER_STDERR_BYTES, ); @@ -414,7 +414,7 @@ const boundedCompilerField = ( redact: (value: string, incomplete?: boolean) => string, ): { readonly value: string | number | null; readonly truncated: boolean } => { if (typeof value !== "string") return { value, truncated: false }; - const bounded = boundedUtf8Text(redact(value), MAX_CAUSE_FIELD_BYTES); + const bounded = utf8HeadExcerpt(redact(value), MAX_CAUSE_FIELD_BYTES); return { value: bounded.text, truncated: bounded.truncated }; }; @@ -440,13 +440,13 @@ const causeLaunchRollback = ( }, }; const result = rollback.result; - const reason = boundedUtf8Text( + const reason = utf8HeadExcerpt( redact(result.reason), MAX_CAUSE_MESSAGE_BYTES, ); const diagnostic = (value: string | undefined): Record => { if (value === undefined) return {}; - const bounded = boundedUtf8Text(redact(value), MAX_CAUSE_MESSAGE_BYTES); + const bounded = utf8HeadExcerpt(redact(value), MAX_CAUSE_MESSAGE_BYTES); return { diagnostic: bounded.text, ...(bounded.truncated ? { diagnostic_truncated: true } : {}), @@ -533,11 +533,11 @@ const causeLaunchOutcome = ( if (cause.cause instanceof GhidraLaunchSupportError) return launchSupport; const output = cause.capturedOutput; if (output === undefined) return launchSupport; - const stdout = boundedUtf8Text( + const stdout = utf8HeadExcerpt( redact(output.stdout), GHIDRA_LAUNCH_COMMAND_OUTPUT_BYTES, ); - const stderr = boundedUtf8Text( + const stderr = utf8HeadExcerpt( redact(output.stderr), GHIDRA_LAUNCH_COMMAND_OUTPUT_BYTES, ); diff --git a/src/native/CommandRunner.ts b/src/native/CommandRunner.ts index a9a46c3a59..aed9d2c9cd 100644 --- a/src/native/CommandRunner.ts +++ b/src/native/CommandRunner.ts @@ -17,7 +17,7 @@ import type { ProviderProcessSupervisor, SpawnedOwnedProviderProcess, } from "../process/ProviderProcess.js"; -import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; import { DEFAULT_NATIVE_COMMAND_TIMEOUT_MS } from "../domain/native/nativeDeadlines.js"; import { selectArtifactCommandPath, @@ -482,11 +482,11 @@ export const failureCapture = ( capture: ProcessCapture, truncated: boolean, ): NativeCommandFailureCapture => { - const stdout = boundedUtf8Text( + const stdout = utf8HeadExcerpt( capture.stdout, NATIVE_FAILURE_DIAGNOSTIC_BYTES, ); - const stderr = boundedUtf8Text( + const stderr = utf8HeadExcerpt( capture.stderr, NATIVE_FAILURE_DIAGNOSTIC_BYTES, ); diff --git a/src/process/DarwinProcessRunTokenReader.ts b/src/process/DarwinProcessRunTokenReader.ts index 97373bd3ad..1687843934 100644 --- a/src/process/DarwinProcessRunTokenReader.ts +++ b/src/process/DarwinProcessRunTokenReader.ts @@ -25,7 +25,7 @@ import type { } from "../domain/analysisErrorBase.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { safeParseJson } from "../domain/safeJson.js"; -import { boundedUtf8Text } from "./boundedUtf8Text.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; import { createDarwinHelperCache } from "./DarwinHelperCache.js"; import { snapshotEnvironment } from "./snapshotEnvironment.js"; import { @@ -266,8 +266,8 @@ const compilerFailureFacts = ( | undefined => { const failure = execFileOutputFailure(cause); if (failure === undefined) return undefined; - const stderr = boundedUtf8Text(failure.stderr, MAX_COMPILER_OUTPUT_BYTES); - const stdout = boundedUtf8Text(failure.stdout, MAX_COMPILER_OUTPUT_BYTES); + const stderr = utf8HeadExcerpt(failure.stderr, MAX_COMPILER_OUTPUT_BYTES); + const stdout = utf8HeadExcerpt(failure.stdout, MAX_COMPILER_OUTPUT_BYTES); const outcome = failure.code === "ETIMEDOUT" ? "timed_out" @@ -336,7 +336,7 @@ const compilerInspectionError = ( code === "ENOENT" ? `macOS process ownership inspection requires the Apple Swift compiler via ${launcher}` : `macOS process ownership helper compilation failed via ${launcher}`; - const stderr = boundedUtf8Text(outcome?.stderr ?? "", STDERR_EXCERPT_LIMIT); + const stderr = utf8HeadExcerpt(outcome?.stderr ?? "", STDERR_EXCERPT_LIMIT); const diagnostics = stderr.text.trim(); const truncated = stderr.truncated || diff --git a/src/process/boundedUtf8Text.ts b/src/process/boundedUtf8Text.ts deleted file mode 100644 index 5915beb65f..0000000000 --- a/src/process/boundedUtf8Text.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** Keep a UTF-8 string within a byte budget without cutting a code point. */ -export const boundedUtf8Text = ( - value: string, - maximumBytes: number, -): { readonly text: string; readonly truncated: boolean } => { - let bytes = 0; - let end = 0; - for (const codePoint of value) { - const codePointBytes = Buffer.byteLength(codePoint, "utf8"); - if (bytes + codePointBytes > maximumBytes) - return { text: value.slice(0, end), truncated: true }; - bytes += codePointBytes; - end += codePoint.length; - } - return { text: value, truncated: false }; -}; diff --git a/src/rizin/RizinDebugSessionManager.ts b/src/rizin/RizinDebugSessionManager.ts index 5dc1335797..5578507e7b 100644 --- a/src/rizin/RizinDebugSessionManager.ts +++ b/src/rizin/RizinDebugSessionManager.ts @@ -62,6 +62,7 @@ import { } from "../process/ownedDebuggerSession.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { ABORTED, waitForAbortable } from "../process/AbortablePromise.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; const CLEANUP_COMMAND_TIMEOUT_MS = 1_000; const STARTUP_TIMEOUT_MS = 30_000; @@ -1287,7 +1288,7 @@ export class RizinDebugSessionManager { const protocolOutput = [...session.history, partialFrame] .filter((frame) => frame.length > 0) .join("\n"); - const stdout = truncateUtf8(protocolOutput, MAX_DIAGNOSTIC_BYTES); + const stdout = utf8HeadExcerpt(protocolOutput, MAX_DIAGNOSTIC_BYTES).text; return { stdout, stderr: output.stderr.text, @@ -1317,7 +1318,10 @@ export class RizinDebugSessionManager { synchronization.resolve(); return; } - const historyPrefix = truncateUtf8(frame.text, MAX_RECENT_FRAME_BYTES); + const historyPrefix = utf8HeadExcerpt( + frame.text, + MAX_RECENT_FRAME_BYTES, + ).text; const historyTruncated = historyPrefix.length < frame.text.length || frame.truncated; if (historyTruncated) session.historyTruncated = true; @@ -1906,7 +1910,7 @@ const waitForCommandFrames = async ( const bytes = Buffer.byteLength(text, "utf8"); if (bytes > remaining) { noteLimit("command_output_bytes"); - const retained = truncateUtf8(text, remaining); + const retained = utf8HeadExcerpt(text, remaining).text; if (retained.length > 0) output.push(retained); outputBytes = maxCommandOutputBytes; } else { @@ -2012,15 +2016,6 @@ const waitForCommandFrames = async ( } }; -const truncateUtf8 = (value: string, maximumBytes: number): string => { - if (maximumBytes <= 0) return ""; - // A UTF-16 code unit needs at most three UTF-8 bytes. Buffer.write stops - // before a partial character; only decode bytes it actually initialized. - const bytes = Buffer.allocUnsafe(Math.min(value.length * 3, maximumBytes)); - const written = bytes.write(value, "utf8"); - return bytes.toString("utf8", 0, written); -}; - const waitForFrame = ( session: RizinDebugSession, timeoutMs: number, diff --git a/src/wasm/WabtFailures.ts b/src/wasm/WabtFailures.ts index ba981e725a..e5fbea54b2 100644 --- a/src/wasm/WabtFailures.ts +++ b/src/wasm/WabtFailures.ts @@ -6,7 +6,7 @@ import { artifactFilePrimaryFacts, bindArtifactFileCleanup, } from "../artifacts/ArtifactFileOperation.js"; -import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import { utf8HeadExcerpt } from "../domain/utf8HeadExcerpt.js"; import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; import { AnalysisError, @@ -88,9 +88,9 @@ export const wabtConfigurationRecoveryAction = (cause?: unknown): string => { }; /** Past the output budget, retained text is a prefix of an oversized stream; keep a bounded excerpt of each. */ const outputLimitExcerpt = (snapshot: ProviderProcessSnapshot) => ({ - stdout: boundedUtf8Text(snapshot.stdout.text, OUTPUT_LIMIT_EXCERPT_BYTES) + stdout: utf8HeadExcerpt(snapshot.stdout.text, OUTPUT_LIMIT_EXCERPT_BYTES) .text, - stderr: boundedUtf8Text(snapshot.stderr.text, OUTPUT_LIMIT_EXCERPT_BYTES) + stderr: utf8HeadExcerpt(snapshot.stderr.text, OUTPUT_LIMIT_EXCERPT_BYTES) .text, // The command reports output-limit only after retention was truncated. truncated: true,