diff --git a/docs/development/testing.md b/docs/development/testing.md index 29f1dabeb7..73f72ce106 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -212,9 +212,9 @@ directives are errors. `vitest/valid-expect` warns (`maxArgs: 2`) because valid collected assertions can be awaited later. `import/no-duplicates` warns with `preferInline: false`. -`no-useless-escape` keeps intentional `\-` regex escapes because advertised -patterns must also compile in `v` mode; the catalog regression checks every -advertised pattern. +`no-useless-escape` keeps intentional `\-` regex escapes. Source regexes need +no escaping for clients: the presentation re-emits every advertised pattern in +the portable grammar (`src/contracts/portablePattern.ts`). Fast lint stays syntax-based. `npm run lint:typed` runs serially with pinned `oxlint-tsgolint` 7.0.2005 and records native plus typed diagnostics in @@ -294,7 +294,17 @@ advertised dialect: the keyword allowlist, size and depth limits, and the regex grammar that compiles in V8, Python, RE2 and Rust. `advertisedDialect.test.ts` lists every input, output and catalog profile combination and compares the violation counts with `advertisedDialectBaseline.ts`, which may only shrink; -the packaged catalog check applies the same rules to the default profiles. Keep canonical rejection checks separate from the +the packaged catalog check applies the same rules to the default profiles. +Advertised patterns, input and output, are enforced at zero: +`portablePattern.ts` re-emits each one exactly, widens it, or omits it, and the +presentation drops any enclosing `not` or `if` and turns `oneOf` into `anyOf` +when a pattern widens. `portablePattern.test.ts` pins the same acceptance of +BMP strings in V8's plain, `u` and `v` modes. Python, Go and Rust read code +points as `u` does; Python's `$` also matches before a final newline, which +only widens. A client compiling without `u` may still reject one astral +character where a wildcard, shorthand or negated class must match it (for +example `^[\S]$`); `PortablePatternProjection` documents this accepted +limitation. Keep canonical rejection checks separate from the intentionally reduced compact advertisement, and probe actual client renderers when changing schema representations. diff --git a/scripts/lib/verify-package-core.mjs b/scripts/lib/verify-package-core.mjs index 5487b28688..ceb27fd5e9 100644 --- a/scripts/lib/verify-package-core.mjs +++ b/scripts/lib/verify-package-core.mjs @@ -14,16 +14,14 @@ export const verifyCompleteToolCatalog = async ( { requireMcpLifecycleResult, sortVerifierNames }, { TOOL_CONTRACTS }, dialect, - { ADVERTISED_INPUT_DIALECT_BASELINE, ADVERTISED_OUTPUT_DIALECT_BASELINE }, + { ADVERTISED_INPUT_DIALECT_BASELINE }, { mcpInputSchemaProfileSchema }, - { mcpOutputSchemaProfileSchema }, ] = await Promise.all([ import("./mcp-verifier-results.mjs"), import("../../dist/contracts/toolContracts.js"), import("../../dist/contracts/advertisedDialect.js"), import("../../dist/contracts/advertisedDialectBaseline.js"), import("../../dist/config/mcpInputSchemaProfile.js"), - import("../../dist/config/mcpOutputSchemaProfile.js"), ]); const listed = await client.listTools(undefined, options); const canonical = sortVerifierNames(TOOL_CONTRACTS.map(({ name }) => name)); @@ -54,9 +52,6 @@ export const verifyCompleteToolCatalog = async ( const inputProfile = mcpInputSchemaProfileSchema.parse( environment.REA_MCP_INPUT_SCHEMA_PROFILE ?? "compact", ); - const outputProfile = mcpOutputSchemaProfileSchema.parse( - environment.REA_MCP_OUTPUT_SCHEMA_PROFILE ?? "full", - ); const regressions = [ ...dialect.advertisedDialectRegressions( dialect.countAdvertisedDialectViolations( @@ -68,7 +63,7 @@ export const verifyCompleteToolCatalog = async ( dialect.countAdvertisedDialectViolations( listed.tools.flatMap(dialect.advertisedOutputPatternViolations), ), - ADVERTISED_OUTPUT_DIALECT_BASELINE[outputProfile], + {}, ), ]; if (regressions.length > 0) diff --git a/src/contracts/advertisedDialectBaseline.ts b/src/contracts/advertisedDialectBaseline.ts index 12a8286a5f..a02c27a7c3 100644 --- a/src/contracts/advertisedDialectBaseline.ts +++ b/src/contracts/advertisedDialectBaseline.ts @@ -6,6 +6,9 @@ // One exception grows them: each runtime analysis setting adds a branch to // update_analysis_settings with two `const` members (`setting`, `action`). +// Every advertised pattern, input and output, is enforced at zero: the +// presentation re-emits each one in the portable grammar. + /** Input-side counts, keyed by `REA_MCP_INPUT_SCHEMA_PROFILE`. */ export const ADVERTISED_INPUT_DIALECT_BASELINE: Readonly< Record<"compact" | "full", Readonly>> @@ -24,9 +27,6 @@ export const ADVERTISED_INPUT_DIALECT_BASELINE: Readonly< "input keyword prefixItems": 2, "input keyword propertyNames": 3, "input keyword uniqueItems": 2, - "input pattern escape:\\/": 12, - "input pattern shorthand-class": 21, - "input pattern unanchored": 9, "input type type-list": 11, "input type type:null": 14, "input type untyped": 40, @@ -52,33 +52,13 @@ export const ADVERTISED_INPUT_DIALECT_BASELINE: Readonly< "input keyword if": 7, "input keyword minProperties": 12, "input keyword not": 10, - "input keyword oneOf": 223, + "input keyword oneOf": 219, "input keyword prefixItems": 255, "input keyword propertyNames": 53, "input keyword then": 7, "input keyword uniqueItems": 5, - "input pattern dot": 5, - "input pattern escape:\\/": 105, - "input pattern non-ascii-or-braced-hex-escape": 1, - "input pattern shorthand-class": 30, - "input pattern unanchored": 98, - "input pattern unescaped-syntax-in-class:^": 1, "input type type-list": 51, "input type type:null": 372, "input type untyped": 1454, }, }; - -/** Output-schema pattern counts, keyed by `REA_MCP_OUTPUT_SCHEMA_PROFILE`. */ -export const ADVERTISED_OUTPUT_DIALECT_BASELINE: Readonly< - Record<"full" | "compact" | "off", Readonly>> -> = { - full: { - "output pattern dot": 7, - "output pattern escape:\\/": 7, - "output pattern shorthand-class": 31, - "output pattern unanchored": 29, - }, - compact: {}, - off: {}, -}; diff --git a/src/contracts/canonicalBase64Metadata.test.ts b/src/contracts/canonicalBase64Metadata.test.ts index fb5aea42fe..91578256f6 100644 --- a/src/contracts/canonicalBase64Metadata.test.ts +++ b/src/contracts/canonicalBase64Metadata.test.ts @@ -53,13 +53,15 @@ it.each(["draft-2020-12", "draft-07"] as const)( }); const Validator = target === "draft-2020-12" ? Ajv2020 : Ajv; const validate = new Validator({ strict: false }).compile(projected); - for (const [encoded, valid] of [ - ["AA==", true], - ["AB==", false], - ["AAAAAA==", false], + // The unbounded pad-bit probe has no stack-safe portable form, so the + // advertisement admits `AB==` while the server still refuses it. + for (const [encoded, valid, advertised] of [ + ["AA==", true, true], + ["AB==", false, true], + ["AAAAAA==", false, false], ] as const) { const value = { encoded, literal }; - expect(validate(value), encoded).toBe(valid); + expect(validate(value), encoded).toBe(advertised); expect(input.safeParse(value).success, encoded).toBe(valid); expect(output.safeParse(value).success, encoded).toBe(valid); } diff --git a/src/contracts/inputSchemaPresentation.ts b/src/contracts/inputSchemaPresentation.ts index 275eb800f7..8f50f9279b 100644 --- a/src/contracts/inputSchemaPresentation.ts +++ b/src/contracts/inputSchemaPresentation.ts @@ -4,10 +4,13 @@ import { } from "./jsonSchemaReferences.js"; import { isDeepStrictEqual } from "node:util"; +import { advertisedSchemaPatterns } from "./advertisedDialect.js"; +import { projectPortablePattern } from "./portablePattern.js"; import { schemaArrays, schemaChildren, schemaMaps, + schemaPointerSteps, schemaSubschemas, } from "./jsonSchemaKeywords.js"; @@ -170,8 +173,17 @@ type SchemaNormalization = { readonly changed: boolean; }; +// A branch that also references another schema is not judged by its own +// keywords: draft-07 ignores a reference's siblings, and the target may admit +// more than a boolean. Such unions keep the `anyOf` fallback. +const hasReference = (schema: Readonly>): boolean => + ["$ref", "$dynamicRef", "$recursiveRef"].some( + (keyword) => typeof schema[keyword] === "string", + ); + const isBooleanLiteralSchema = (schema: unknown): boolean => isObject(schema) && + !hasReference(schema) && (typeof schema.const === "boolean" || (Array.isArray(schema.enum) && schema.enum.length > 0 && @@ -179,7 +191,17 @@ const isBooleanLiteralSchema = (schema: unknown): boolean => const isBooleanSchema = (schema: unknown): boolean => isObject(schema) && - (schema.type === "boolean" || isBooleanLiteralSchema(schema)); + ((schema.type === "boolean" && !hasReference(schema)) || + isBooleanLiteralSchema(schema)); + +/** + * Whether a union collapses to `type: "boolean"` once a branch changes. The + * decision reads the canonical branches, so a branch that becomes boolean only + * because presentation relaxed its reference keeps the union, and pointers into + * the union stay resolvable (see `throughRemovableLocation`). + */ +const collapsesToBoolean = (branches: readonly unknown[]): boolean => + branches.length > 0 && branches.every(isBooleanSchema); const isObjectKeyword = (key: string) => schemaMaps.has(key); const isArrayKeyword = (key: string) => schemaArrays.has(key); @@ -247,9 +269,10 @@ const referencedDefaultPaths = ( const kept = new Set(); for (const { target } of new JsonSchemaReferences(schema).references()) { if (target.kind !== "target") continue; - const defaultAt = target.path.indexOf("default"); - if (defaultAt < 0) continue; - kept.add(JSON.stringify(target.path.slice(0, defaultAt + 1))); + const steps = schemaPointerSteps(target.path); + const last = steps.at(-1); + if (last?.kind !== "value" || last.keyword !== "default") continue; + kept.add(JSON.stringify(target.path.slice(0, steps.length))); } return kept; }; @@ -278,8 +301,93 @@ const hasNormalizedLiteral = (schema: Readonly>) => schema.enum.every((item) => typeof item === "boolean")) || schema.enum.includes(""))); +/** + * A pattern the portable projection cannot reproduce exactly widens its site. + * Exact re-emission keeps the site's meaning, so its dependents stay intact. + */ +const hasWidenedPattern = (schema: Readonly>) => + typeof schema.pattern === "string" && + projectPortablePattern(schema.pattern).kind !== "exact"; + +interface NormalizationOptions { + /** Rewrite boolean and empty-string literals some gateways refuse. */ + readonly literals: boolean; +} +const INPUT_NORMALIZATION: NormalizationOptions = { literals: true }; +const OUTPUT_NORMALIZATION: NormalizationOptions = { literals: false }; + /** One projection owns actual rewrite sites and their scoped validation dependencies. */ -const normalizationContext = (root: Readonly>) => { +/** + * Keywords normalization may remove or rename: a widened `not` or `if` is + * dropped with its branches, a widened `oneOf` becomes `anyOf`, and lost + * evaluation drops `unevaluatedProperties` and `unevaluatedItems`. + */ +const REMOVABLE_KEYWORDS = new Set([ + "oneOf", + "not", + "if", + "then", + "else", + "unevaluatedProperties", + "unevaluatedItems", +]); + +/** + * Whether a pointer from `root` passes through a keyword normalization may + * remove. An `anyOf` is removed only when it collapses: every branch boolean. + */ +/** Keywords a relaxed draft-07 reference keeps; its other siblings go. */ +const DRAFT07_REDUCED_REFERENCE_KEYS = new Set([ + "$id", + "$schema", + "$anchor", + "$defs", + "definitions", + "description", + "title", + "default", + "examples", + "$comment", +]); + +const throughRemovableLocation = ( + root: unknown, + path: readonly string[], + draft07: boolean, +): boolean => { + let value = root; + for (const [index, step] of schemaPointerSteps(path).entries()) { + // Relaxing a draft-07 reference removes its ignored siblings, including + // literal-valued ones such as `const`. + if ( + draft07 && + step.kind !== "entry" && + isObject(value) && + hasReference(value) && + !DRAFT07_REDUCED_REFERENCE_KEYS.has(step.keyword) + ) + return true; + if ( + step.kind === "keyword" && + (REMOVABLE_KEYWORDS.has(step.keyword) || + (step.keyword === "anyOf" && + isObject(value) && + Array.isArray(value.anyOf) && + collapsesToBoolean(value.anyOf))) + ) + return true; + value = + typeof value === "object" && value !== null + ? Reflect.get(value, path[index]!) + : undefined; + } + return false; +}; + +const normalizationContext = ( + root: Readonly>, + options: NormalizationOptions, +) => { const references = new JsonSchemaReferences(root); type Fact = { changed: boolean; @@ -318,6 +426,7 @@ const normalizationContext = (root: Readonly>) => { fact: Fact; }[] = []; const changed: Fact[] = []; + const untracked: Fact[] = []; const factAt = ( schema: unknown, inherited: JsonSchemaScope | undefined, @@ -343,17 +452,34 @@ const normalizationContext = (root: Readonly>) => { const { schema, scope, fact } = work[index]!; const ignoresSiblings = scope?.dialect === "draft-07" && typeof schema.$ref === "string"; + const projection = + typeof schema.pattern === "string" + ? projectPortablePattern(schema.pattern) + : undefined; + const opaquePatternChanged = + !sites.get(scope)?.has(schema) && + projection !== undefined && + (projection.kind !== "exact" || projection.pattern !== schema.pattern); if ( !ignoresSiblings && - sites.get(scope)?.has(schema) && - hasNormalizedLiteral(schema) + (opaquePatternChanged || + (sites.get(scope)?.has(schema) && + ((options.literals && hasNormalizedLiteral(schema)) || + hasWidenedPattern(schema)))) ) { fact.changed = true; changed.push(fact); } if (typeof schema.$ref === "string") { const target = references.resolve(scope, schema.$ref); - if (target.kind === "target") { + if ( + target.kind === "target" && + !throughRemovableLocation( + scope?.root, + target.path, + scope?.dialect === "draft-07", + ) + ) { const dependency = factAt(target.value, scope); dependency?.dependents.add(fact); dependency?.evaluationDependents.add(fact); @@ -363,8 +489,24 @@ const normalizationContext = (root: Readonly>) => { !sites.get(references.enter(target.value, scope))?.has(target.value) ) fact.opaqueReference = dependency; - } + } else if (target.kind !== "unresolved") { + // Named anchors and pointers into removable locations are not + // followed by this owner. Relax them when normalizing so a widened or + // moved target cannot make a negation stricter or leave a dangling + // reference. + fact.opaqueReference = fact; + if (!fact.changed) { + fact.changed = true; + changed.push(fact); + } + } else untracked.push(fact); } + if ( + scope?.dialect !== "draft-07" && + (typeof schema.$dynamicRef === "string" || + typeof schema.$recursiveRef === "string") + ) + untracked.push(fact); if (!ignoresSiblings) for (const child of schemaSubschemas(schema, { includeDefinitions: false, @@ -391,6 +533,17 @@ const normalizationContext = (root: Readonly>) => { factAt(child, scope)?.evaluationDependents.add(fact); } } + // Dynamic targets depend on the evaluation path, and an unresolved target + // may be an embedded resource this owner rewrote. Neither is followed, so + // once anything widens, relax them like named anchors. + if (changed.length > 0) + for (const fact of untracked) { + fact.opaqueReference = fact; + if (!fact.changed) { + fact.changed = true; + changed.push(fact); + } + } for (let index = 0; index < changed.length; index++) for (const dependent of changed[index]!.dependents) if (!dependent.changed) { @@ -416,6 +569,7 @@ const normalizationContext = (root: Readonly>) => { } return { references, + options, changed: (schema: object, scope: JsonSchemaScope | undefined) => indexed.get(scope)?.get(schema)?.changed === true, lostEvaluation: (schema: object, scope: JsonSchemaScope | undefined) => @@ -425,19 +579,51 @@ const normalizationContext = (root: Readonly>) => { }; }; +/** Drop boolean and empty-string literals, keeping the node's type. */ +const widenGatewayLiterals = (node: Record): void => { + if (typeof node.const === "boolean") { + delete node.const; + if (node.type === undefined) node.type = "boolean"; + } + if (node.const === "") { + delete node.const; + if (node.type === undefined) node.type = "string"; + } + if ( + Array.isArray(node.enum) && + node.enum.length > 0 && + node.enum.every((item) => typeof item === "boolean") + ) { + delete node.enum; + if (node.type === undefined) node.type = "boolean"; + } + if (Array.isArray(node.enum) && node.enum.includes("")) { + // Removing only the empty member would reject a canonical-valid input. + // Gemini transports turn const into enum and refuse empty enum members. + const values = node.enum; + delete node.enum; + if ( + node.type === undefined && + values.every((item) => typeof item === "string") + ) + node.type = "string"; + } +}; + const normalizeAdvertisedSchema = ( value: unknown, + options: NormalizationOptions, context?: ReturnType, inherited?: JsonSchemaScope, ): SchemaNormalization => { if (!isObject(value)) return { schema: value, changed: false }; - const owner = context ?? normalizationContext(value); + const owner = context ?? normalizationContext(value, options); const scope = owner.references.enter( value, context === undefined ? owner.references.rootScope : inherited, ); const normalize = (child: unknown) => - normalizeAdvertisedSchema(child, owner, scope); + normalizeAdvertisedSchema(child, options, owner, scope); const node: Record = { ...value }; const changed = owner.changed(value, scope); const normalizeMap = (map: Readonly>) => @@ -484,33 +670,15 @@ const normalizeAdvertisedSchema = ( } } - if (typeof node.const === "boolean") { - delete node.const; - if (node.type === undefined) node.type = "boolean"; - } - if (node.const === "") { - delete node.const; - if (node.type === undefined) node.type = "string"; - } - if ( - Array.isArray(node.enum) && - node.enum.length > 0 && - node.enum.every((item) => typeof item === "boolean") - ) { - delete node.enum; - if (node.type === undefined) node.type = "boolean"; - } - if (Array.isArray(node.enum) && node.enum.includes("")) { - // Removing only the empty member would reject a canonical-valid input. - // Gemini transports turn const into enum and refuse empty enum members. - const values = node.enum; - delete node.enum; - if ( - node.type === undefined && - values.every((item) => typeof item === "string") - ) - node.type = "string"; + if (typeof node.pattern === "string") { + // Clients refuse the complete tools request over one pattern their regex + // engine cannot compile. A widened or omitted pattern marks this site + // changed, so enclosing `not`, `if` and `oneOf` keep only widening. + const projected = projectPortablePattern(node.pattern); + if (projected.kind === "unexpressible") delete node.pattern; + else node.pattern = projected.pattern; } + if (options.literals) widenGatewayLiterals(node); const negated = Object.hasOwn(value, "not") ? normalize(value.not) @@ -571,11 +739,7 @@ const normalizeAdvertisedSchema = ( } continue; } - if ( - branchesChanged && - schemas.length > 0 && - schemas.every(isBooleanSchema) - ) { + if (branchesChanged && collapsesToBoolean(branches)) { node.type = "boolean"; preserveSharedDefault(node, schemas); delete node[combinator]; @@ -593,22 +757,12 @@ const normalizeAdvertisedSchema = ( // this affected reference rather than copy or mutate its executable view. if (scope?.dialect === "draft-07") { for (const key of Object.keys(node)) - if ( - ![ - "$id", - "$schema", - "$anchor", - "$defs", - "definitions", - "description", - "title", - "default", - "examples", - "$comment", - ].includes(key) - ) - delete node[key]; - } else delete node.$ref; + if (!DRAFT07_REDUCED_REFERENCE_KEYS.has(key)) delete node[key]; + } else { + delete node.$ref; + delete node.$dynamicRef; + delete node.$recursiveRef; + } node.description = [ node.description, "This referenced input schema is advertised in reduced form; the server validates the complete canonical input.", @@ -855,10 +1009,8 @@ export const presentInputJsonSchema = ( describeProperty, ); if (!isObject(described)) - return normalizeAdvertisedSchema(flattened).schema as Record< - string, - unknown - >; + return normalizeAdvertisedSchema(flattened, INPUT_NORMALIZATION) + .schema as Record; const { dependencies: _dependencies, dependentRequired: _dependentRequired, @@ -881,18 +1033,99 @@ export const presentInputJsonSchema = ( : [], ); if (dependencyGuidance.length === 0) - return normalizeAdvertisedSchema(presentation).schema as Record< - string, - unknown - >; + return normalizeAdvertisedSchema(presentation, INPUT_NORMALIZATION) + .schema as Record; const description = [ typeof presentation.description === "string" ? presentation.description : undefined, ...dependencyGuidance, ].filter((text) => text !== undefined); - return normalizeAdvertisedSchema({ - ...presentation, - description: description.join(" "), - }).schema as Record; + return normalizeAdvertisedSchema( + { ...presentation, description: description.join(" ") }, + INPUT_NORMALIZATION, + ).schema as Record; +}; + +/** + * Present an output schema with portable patterns. SDK clients compile every + * listed output schema; literals stay, since no model reads them. + */ +export const presentOutputJsonSchema = ( + root: Record, +): Record => { + const patterns = advertisedSchemaPatterns(root).map(({ pattern }) => ({ + pattern, + projection: projectPortablePattern(pattern), + })); + if (hasLiteralReference(root)) + return normalizeAdvertisedSchema(root, OUTPUT_NORMALIZATION) + .schema as Record; + // Output schemas reach several megabytes. Exact re-emission changes no + // dependent, so only a widened pattern needs the dependency analysis. + if (patterns.some(({ projection }) => projection.kind !== "exact")) + return normalizeAdvertisedSchema(root, OUTPUT_NORMALIZATION) + .schema as Record; + if ( + patterns.every( + ({ pattern, projection }) => + projection.kind === "exact" && projection.pattern === pattern, + ) + ) + return root; + return replaceExactPatterns(root) as Record; +}; + +/** A reference into literal data needs dependency analysis, not in-place rewriting. */ +const hasLiteralReference = (root: Record): boolean => { + const pending: unknown[] = [root]; + const visited = new WeakSet(); + while (pending.length > 0) { + const value = pending.pop(); + if (!isObject(value) || visited.has(value)) continue; + visited.add(value); + if (typeof value.$ref === "string") { + try { + const reference = decodeURIComponent(value.$ref); + if ( + /(?:^#\/|\/)(?:default|const|enum|examples)(?:\/|$)/u.test(reference) + ) + return true; + } catch { + // An invalid fragment cannot name one of these locations. + } + } + pending.push(...schemaSubschemas(value, { includeDefinitions: true })); + } + return false; +}; + +/** Copy schema positions, replacing each pattern with its exact portable form. */ +const replaceExactPatterns = (schema: unknown): unknown => { + if (!isObject(schema)) return schema; + return Object.fromEntries( + Object.entries(schema).map(([key, value]) => { + if (key === "pattern" && typeof value === "string") { + const projection = projectPortablePattern(value); + return [key, projection.kind === "exact" ? projection.pattern : value]; + } + if (schemaMaps.has(key) && isObject(value)) + return [ + key, + Object.fromEntries( + Object.entries(value).map(([name, child]) => [ + name, + replaceExactPatterns(child), + ]), + ), + ]; + if ( + (schemaArrays.has(key) || schemaChildren.has(key)) && + Array.isArray(value) + ) + return [key, value.map(replaceExactPatterns)]; + if (schemaChildren.has(key)) return [key, replaceExactPatterns(value)]; + return [key, value]; + }), + ); }; diff --git a/src/contracts/jsonSchemaKeywords.ts b/src/contracts/jsonSchemaKeywords.ts index ed161ee2b7..62319bb626 100644 --- a/src/contracts/jsonSchemaKeywords.ts +++ b/src/contracts/jsonSchemaKeywords.ts @@ -50,3 +50,46 @@ export const schemaLiteralValues = new Set([ "default", "examples", ]); + +/** One segment of a JSON Pointer into a schema, classified by position. */ +export type SchemaPointerStep = + | { readonly kind: "keyword"; readonly keyword: string } + | { readonly kind: "entry" } + | { readonly kind: "value"; readonly keyword: string }; + +/** + * Classify each segment of a JSON Pointer into a schema by its position, so a + * map entry or property named like a keyword is never read as that keyword. + * The walk ends at the first keyword that is not a subschema keyword (such as + * `default`); the segments after it address instance data and are omitted. + */ +export const schemaPointerSteps = ( + path: readonly string[], +): SchemaPointerStep[] => { + const steps: SchemaPointerStep[] = []; + let position: "schema" | "entry" | "schemaOrIndex" = "schema"; + for (const segment of path) { + if ( + position === "entry" || + (position === "schemaOrIndex" && /^(?:0|[1-9]\d*)$/u.test(segment)) + ) { + steps.push({ kind: "entry" }); + position = "schema"; + continue; + } + if (schemaMaps.has(segment) || schemaArrays.has(segment)) { + steps.push({ kind: "keyword", keyword: segment }); + position = "entry"; + continue; + } + if (schemaChildren.has(segment)) { + steps.push({ kind: "keyword", keyword: segment }); + // `items` and its siblings may hold a draft-07 tuple array. + position = "schemaOrIndex"; + continue; + } + steps.push({ kind: "value", keyword: segment }); + break; + } + return steps; +}; diff --git a/src/contracts/jsonSchemaReferences.ts b/src/contracts/jsonSchemaReferences.ts index 86c2630625..e593ee723d 100644 --- a/src/contracts/jsonSchemaReferences.ts +++ b/src/contracts/jsonSchemaReferences.ts @@ -50,6 +50,27 @@ const dialectOf = ( ? "draft-2020-12" : undefined; +/** + * The fragment of a reference into the current resource: a bare fragment, or + * a URI whose resolution against the resource's absolute base names that same + * resource. Other resources are external. + */ +const sameResourceFragment = ( + reference: string, + baseUri: string | undefined, +): string | undefined => { + if (reference.startsWith("#")) return reference.slice(1); + if (baseUri === undefined || !URL.canParse(reference, baseUri)) + return undefined; + const resource = new URL(reference, baseUri); + const base = new URL(baseUri); + resource.hash = ""; + base.hash = ""; + if (resource.href !== base.href) return undefined; + const hash = reference.indexOf("#"); + return hash === -1 ? "" : reference.slice(hash + 1); +}; + /** Per-projection scope ownership; no external resource loading or dynamic-reference evaluation. */ export class JsonSchemaReferences { readonly rootScope: JsonSchemaScope; @@ -217,11 +238,11 @@ export class JsonSchemaReferences { ): LocalReference { if (scope === undefined || this.enter(scope.root, scope) === undefined) return { kind: "unresolved", reason: "scope" }; - if (!reference.startsWith("#")) - return { kind: "unresolved", reason: "external" }; + const local = sameResourceFragment(reference, scope.baseUri); + if (local === undefined) return { kind: "unresolved", reason: "external" }; let fragment: string; try { - fragment = decodeURIComponent(reference.slice(1)); + fragment = decodeURIComponent(local); } catch { return { kind: "unresolved", reason: "fragment" }; } diff --git a/src/contracts/portablePattern.test.ts b/src/contracts/portablePattern.test.ts new file mode 100644 index 0000000000..cba3c984e6 --- /dev/null +++ b/src/contracts/portablePattern.test.ts @@ -0,0 +1,582 @@ +import { describe, expect, it } from "vitest"; +import { Ajv } from "ajv"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { INVALID_BASE64_PADDING_PATTERN } from "../domain/stringPatterns.js"; + +import { portablePatternViolations } from "./advertisedDialect.js"; +import { + presentInputJsonSchema, + presentOutputJsonSchema, +} from "./inputSchemaPresentation.js"; +import { projectPortablePattern } from "./portablePattern.js"; + +// Short strings over an alphabet that reaches every class boundary the +// projection reasons about: ASCII letters, digits, punctuation, whitespace, +// line terminators, NUL, non-ASCII spaces and letters, and a lone surrogate. +const ALPHABET = [ + "a", + "Z", + "0", + "7", + "_", + "-", + "/", + "\\", + ".", + "=", + "+", + "^", + "$", + "[", + " ", + "\t", + "\n", + "\r", + "\0", + " ", + "
", + "é", + "😀", + "\ud800", + "A", + "Q", + "x", + "c", + "o", + "n", +]; + +const samples = (() => { + const strings = new Set([""]); + let state = 0x5eed; + const next = () => { + state = (state * 1103515245 + 12345) % 2 ** 31; + return state; + }; + for (const character of ALPHABET) strings.add(character); + for (let index = 0; index < 4000; index++) { + let value = ""; + const length = next() % 9; + for (let position = 0; position < length; position++) + value += ALPHABET[next() % ALPHABET.length]; + strings.add(value); + } + return [...strings]; +})(); +const bmpSamples = samples.filter( + (value) => !/[\u{10000}-\u{10FFFF}]/u.test(value), +); + +const SOURCES = [ + // Advertised before projection (input and output catalogs). + "^\\/", + "^-?\\d+$", + "^[^\\s\\x5b\\x5d]+$", + "^[^\\x00\\s\"']+$", + "^(?:....)*$", + "[^A-Za-z0-9+\\x2f=]", + "(?:=[^=]|===)", + "(?:[^AQgw]==$|[BCDFGHJKLNOPRSTVWXZabdefhijlmnpqrtuvxyz1235679+\\x2f]=$)", + "^1\\.0\\.\\d+$", + "^\\x2F(?:[^\\x2F\\x5C]|$)", + "^(?:\\/.*)?$", + "^[!#$%&'*+.^_`\\x7C~0-9A-Za-z\\x2D]+$", + "^[\\t\\x20-\\x7e\\x80-\\xff]*$", + "^[\\w .:@\\x2f+\\x2d]+$", + "^https?:\\/\\/", + "^INVOKE_", + "^[\\x01-\\x20]*(?:\\+|-)?(?:NaN|Infinity|(?:(?:\\d+(?:\\.\\d*)?|\\.\\d+)(?:[eE](?:\\+|-)?\\d+)?[fFdD]?|0[xX](?:[\\da-fA-F]+(?:\\.[\\da-fA-F]*)?|\\.[\\da-fA-F]+)[pP](?:\\+|-)?\\d+[fFdD]?))[\\x01-\\x20]*$", + // Constructs other engines refuse. + "^[^\\0]*$", + "^[^\\uD800-\\uDFFF]*$", + "^[A-Za-z0-9._-]+$", + "^[a-z0-9]+(?:-[a-z0-9]+)*(?![\\s\\S])", + "^(?!(?:con|prn|aux|nul|com[1-9]|lpt[1-9])$)", + "^(?!\\.\\.?$)[^/]+$", + "^[^]$", + "a|^b|c$", + "^\\p{L}+$", + "^\\bcon\\b", + "^a{2,1001}$", + "^a{01}$", + "^(?:ab){0,02}$", + "^(?[a-z]+)-x$", + "^\\u0041\\u{42}\\cJ$", + "^\\S\\W\\D$", + "^[\\S]$", + "^[^\\D]$", + "^é+$", +]; + +describe("portable pattern projection", () => { + it.each(SOURCES)( + "re-emits %s inside the grammar and its declared relation", + (source) => { + const projection = projectPortablePattern(source); + if (projection.kind === "unexpressible") { + expect( + presentOutputJsonSchema({ type: "string", pattern: source }), + ).not.toHaveProperty("pattern"); + return; + } + expect(portablePatternViolations(projection.pattern)).toEqual([]); + for (const flags of ["", "u", "v"]) + expect(() => new RegExp(projection.pattern, flags)).not.toThrow(); + const original = new RegExp(source, "u"); + const projected = new RegExp(projection.pattern, "u"); + // Engines differ only on astral characters (see `projectPortablePattern`): + // V8 without `u` matches UTF-16 units, the other modes code points. + for (const flags of ["", "v"]) { + const mode = new RegExp(projection.pattern, flags); + expect( + bmpSamples + .filter((value) => mode.test(value) !== projected.test(value)) + .map((value) => JSON.stringify(value)), + `/${projection.pattern}/${flags}`, + ).toEqual([]); + } + const differences = samples.filter( + (value) => original.test(value) !== projected.test(value), + ); + if (projection.kind === "exact") + expect(differences.map((value) => JSON.stringify(value))).toEqual([]); + else + // A wider projection may only admit more strings. + expect( + differences + .filter((value) => original.test(value)) + .map((value) => JSON.stringify(value)), + ).toEqual([]); + }, + ); + + it.each([ + ["^\\d+$", "exact", "^[0-9]+$"], + ["^[A-Za-z0-9._-]+$", "exact", "^[\\-.0-9A-Z_a-z]+$"], + ["^\\/", "exact", "^/[^\\x00]*(?:\\x00[^\\x00]*)*$"], + ["^(?!con$)[a-z]+$", "wider", "^[a-z]+$"], + ["^[^\\x00\\s\"']+$", "wider", "^[^\\x00\\x09-\\x0d \"']+$"], + ])("projects %s %s as %s", (source, kind, pattern) => { + expect(projectPortablePattern(source)).toEqual({ kind, pattern }); + }); + + // An astral literal is two UTF-16 units in V8 without `u`; a one-or-two + // unit atom would backtrack exponentially under a repeat. + it.each([ + "^(a)\\1$", + "^(?a)\\k$", + "^[]$", + "^(?i:a)$", + "^😀$", + "^é😀+$", + "^[😀]$", + "^[a😀]+$", + "^[\\u{1F600}-\\u{1F64F}]$", + "^[^😀]$", + ])("omits %s, which has no portable form", (source) => { + expect(projectPortablePattern(source)).toEqual({ + kind: "unexpressible", + }); + }); + + it("keeps a repeated class with non-ASCII members in one stack-safe class", () => { + const projected = projectPortablePattern("^[\\t\\x20-\\x7e\\x80-\\xff]*$"); + expect(projected).toEqual({ + kind: "wider", + pattern: "^[^\\x00-\\x08\\x0a-\\x1f\\x7f]*$", + }); + if (projected.kind === "unexpressible") throw new Error("Expected a form"); + const large = "A".repeat(8 * 1024 * 1024); + // V8 `v` mode overflows on any long class repetition, the source included. + for (const flags of ["", "u"]) + expect(new RegExp(projected.pattern, flags).test(large)).toBe(true); + }); + + it("omits the unsafe unbounded base64 padding projection", () => { + expect( + projectPortablePattern(INVALID_BASE64_PADDING_PATTERN.source), + ).toEqual({ kind: "unexpressible" }); + }); + + it.each(["^[^^]+$", "^[^^a]+$"])( + "preserves the literal leading caret in %s", + (pattern) => { + const projected = projectPortablePattern(pattern); + expect(projected.kind).toBe("exact"); + if (projected.kind === "unexpressible") + throw new Error("Expected exact projection"); + for (const input of ["abc", "^", "a", "Z"]) + expect(new RegExp(projected.pattern, "u").test(input)).toBe( + new RegExp(pattern, "u").test(input), + ); + }, + ); +}); + +const PRESENTERS = [ + [ + "input", + (schema: Record) => + presentInputJsonSchema(schema, () => undefined), + ], + ["output", presentOutputJsonSchema], +] as const; + +describe("advertised pattern presentation", () => { + const lookAround = "^(?!con$)[a-z]+$"; + const property = (schema: Record) => ({ + type: "object", + properties: { name: schema }, + }); + + it.each(PRESENTERS)( + "keeps only widening around a widened %s pattern", + (_kind, present) => { + // Widening a pattern under `not` would narrow the enclosing schema. + const negated = present( + property({ type: "string", not: { pattern: lookAround } }), + ); + expect(negated).toEqual(property({ type: "string" })); + // Exact re-emission keeps the negation. + expect( + present(property({ type: "string", not: { pattern: "^\\d+$" } })), + ).toEqual(property({ type: "string", not: { pattern: "^[0-9]+$" } })); + // A widened branch could match alongside a sibling, so `oneOf` becomes `anyOf`. + expect( + present( + property({ + oneOf: [ + { type: "string", pattern: lookAround }, + { type: "string", pattern: "^con$" }, + ], + }), + ), + ).toEqual( + property({ + anyOf: [ + { type: "string", pattern: "^[a-z]+$" }, + { type: "string", pattern: "^con$" }, + ], + }), + ); + }, + ); + + it("returns an output schema without non-portable patterns unchanged", () => { + const schema = property({ type: "string", pattern: "^[a-z]+$" }); + expect(presentOutputJsonSchema(schema)).toBe(schema); + }); + + it.each(PRESENTERS)( + "does not narrow %s through a named anchor", + (_kind, present) => { + const schema = { + $defs: { + name: { $anchor: "name", type: "string", pattern: lookAround }, + }, + type: "object", + properties: { name: { not: { $ref: "#name" } } }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ name: "con" })).toBe(true); + }, + ); + + it.each(PRESENTERS)( + "does not narrow %s through a dynamic reference", + (_kind, present) => { + const schema = { + $defs: { + name: { $dynamicAnchor: "name", type: "string", pattern: lookAround }, + }, + type: "object", + properties: { name: { not: { $dynamicRef: "#name" } } }, + }; + // Ajv does not resolve this dynamic anchor, so check the structure: the + // negation of a widened target cannot remain. + const presented = present(schema); + expect(JSON.stringify(presented)).not.toContain(lookAround); + expect( + (presented.properties as Record).name, + ).not.toHaveProperty("not"); + }, + ); + + it.each(PRESENTERS)( + "does not narrow %s through a URI naming an embedded resource", + (_kind, present) => { + const schema = { + $id: "https://example.test/root", + $defs: { name: { $id: "name", type: "string", pattern: lookAround } }, + type: "object", + properties: { name: { not: { $ref: "https://example.test/name" } } }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ name: "con" })).toBe(true); + }, + ); + + it.each(PRESENTERS)( + "does not narrow %s through a URI naming this resource", + (_kind, present) => { + const schema = { + $id: "https://example.test/root", + $defs: { name: { type: "string", pattern: lookAround } }, + type: "object", + properties: { + name: { not: { $ref: "https://example.test/root#/$defs/name" } }, + }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ name: "con" })).toBe(true); + }, + ); +}); + +describe("advertised pattern presentation through pointers", () => { + const lookAround = "^(?!con$)[a-z]+$"; + + it.each(PRESENTERS)( + "keeps %s pointers valid into locations presentation may remove", + (_kind, present) => { + // A widened `oneOf` becomes `anyOf`; a widened `not` or `if` is dropped. + const widened = { type: "string", pattern: lookAround }; + const schema = { + $defs: { + choice: { + oneOf: [ + { type: "string", pattern: "^\\s+$" }, + { type: "string", pattern: "^ok$" }, + ], + }, + guard: { not: widened }, + gate: { if: widened, then: { type: "string" }, else: widened }, + }, + type: "object", + properties: { + choice: { $ref: "#/$defs/choice/oneOf/1" }, + negated: { $ref: "#/$defs/guard/not" }, + predicate: { $ref: "#/$defs/gate/if" }, + consequent: { $ref: "#/$defs/gate/then" }, + alternative: { $ref: "#/$defs/gate/else" }, + }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect( + validate({ + choice: "ok", + negated: "con", + predicate: "y", + consequent: "z", + alternative: "w", + }), + ).toBe(true); + }, + ); + + it.each(PRESENTERS)( + "reads %s pointer segments by schema position, not by spelling", + (_kind, present) => { + // `properties` and `items` here are definition names, so the `not` + // after each is the removable keyword, not a map entry. + const widened = { type: "string", pattern: lookAround }; + const schema = { + $defs: { properties: { not: widened }, items: { not: widened } }, + type: "object", + properties: { + first: { $ref: "#/$defs/properties/not" }, + second: { $ref: "#/$defs/items/not" }, + }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ first: "con", second: "con" })).toBe(true); + }, + ); +}); + +it.each(PRESENTERS)( + "keeps %s pointers valid into unevaluated keywords lost evaluation removes", + (_kind, present) => { + const schema = { + $defs: { + guard: { + if: { pattern: "^(?!con$)[a-z]+$" }, + then: { type: "string" }, + unevaluatedProperties: { type: "string" }, + unevaluatedItems: { type: "string" }, + }, + }, + type: "object", + properties: { + props: { $ref: "#/$defs/guard/unevaluatedProperties" }, + items: { $ref: "#/$defs/guard/unevaluatedItems" }, + }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ props: "a", items: "b" })).toBe(true); + }, +); + +it.each(PRESENTERS)( + "keeps %s pointers valid into draft-07 reference siblings relaxation removes", + (_kind, present) => { + const schema = { + $schema: "http://json-schema.org/draft-07/schema#", + definitions: { + word: { $id: "https://example.test/word", type: "object" }, + }, + type: "object", + properties: { + a: { + $ref: "https://example.test/word", + properties: { kept: { type: "string" } }, + }, + b: { $ref: "#/properties/a/properties/kept" }, + name: { type: "string", pattern: "^(?!con$)[a-z]+$" }, + }, + }; + const validate = new Ajv({ strict: false }).compile(present(schema)); + expect(validate({ a: {}, b: "x", name: "con" })).toBe(true); + }, +); + +it.each(PRESENTERS)( + "keeps %s chained pointers valid into removed draft-07 literal siblings", + (_kind, present) => { + const schema = { + $schema: "http://json-schema.org/draft-07/schema#", + definitions: { word: { type: "object" } }, + type: "object", + properties: { + a: { + $ref: "#/definitions/word", + properties: { kept: { type: "string" } }, + }, + b: { + $ref: "#/properties/a/properties/kept", + const: { type: "string" }, + }, + c: { $ref: "#/properties/b/const" }, + name: { type: "string", pattern: "^(?!con$)[a-z]+$" }, + }, + }; + const validate = new Ajv({ strict: false }).compile(present(schema)); + expect(validate({ a: {}, b: "x", c: "y", name: "con" })).toBe(true); + }, +); + +describe("advertised pattern presentation through collapsing unions", () => { + const lookAround = "^(?!con$)[a-z]+$"; + + it.each(PRESENTERS)( + "keeps %s pointers valid into a boolean anyOf that collapses", + (_kind, present) => { + const schema = { + $defs: { + flags: { + anyOf: [ + { type: "boolean", pattern: "^\\s+$" }, + { type: "boolean" }, + ], + }, + }, + type: "object", + properties: { flag: { $ref: "#/$defs/flags/anyOf/0" } }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ flag: true })).toBe(true); + }, + ); + + it.each(PRESENTERS)( + "keeps %s pointers valid into a union whose relaxed reference becomes boolean", + (_kind, present) => { + // Relaxing the reference into `flags` leaves `outer`'s first branch a + // plain boolean; `outer` must still keep the branch `item` targets. + const schema = { + $defs: { + flags: { anyOf: [{ type: "boolean" }, { type: "boolean" }] }, + outer: { + anyOf: [ + { $ref: "#/$defs/flags/anyOf/0", type: "boolean" }, + { type: "boolean" }, + ], + }, + }, + type: "object", + properties: { + item: { $ref: "#/$defs/outer/anyOf/1" }, + name: { type: "string", pattern: lookAround }, + }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ item: true, name: "con" })).toBe(true); + }, + ); + + it.each(PRESENTERS)( + "keeps %s pointers valid into a union whose resource reference relaxes", + (_kind, present) => { + // The widened sibling pattern relaxes the reference to the embedded + // resource, leaving both branches boolean after normalization. + const schema = { + $defs: { + bool: { $id: "https://example.test/bool", type: "boolean" }, + flags: { + anyOf: [ + { $ref: "https://example.test/bool", type: "boolean" }, + { type: "boolean", pattern: "^\\s+$" }, + ], + }, + }, + type: "object", + properties: { flag: { $ref: "#/$defs/flags/anyOf/0" } }, + }; + const validate = new Ajv2020({ strict: false }).compile(present(schema)); + expect(validate({ flag: true })).toBe(true); + }, + ); + + it.each(PRESENTERS)( + "does not collapse %s draft-07 references with ignored literal siblings", + (_kind, present) => { + // Embedded resources are unresolved here, so once a pattern widens the + // references relax; draft-07 ignores each reference's `const` sibling. + const schema = { + $schema: "http://json-schema.org/draft-07/schema#", + definitions: { + word: { $id: "https://example.test/word", type: "string" }, + count: { $id: "https://example.test/count", type: "number" }, + }, + type: "object", + properties: { + value: { + oneOf: [ + { $ref: "https://example.test/word", const: true }, + { $ref: "https://example.test/count", const: false }, + ], + }, + name: { type: "string", pattern: lookAround }, + }, + }; + const create = () => + new Ajv({ strict: false, ignoreKeywordsWithRef: true }); + const canonical = create().compile(schema); + const validate = create().compile(present(schema)); + for (const value of ["word", 1]) { + expect(canonical({ value })).toBe(true); + expect(validate({ value })).toBe(true); + } + }, + ); + + it("relaxes references to nonportable schemas inside literal output data", () => { + const literal = { type: "string", pattern: "^\\d+$" }; + const schema = presentOutputJsonSchema({ + $ref: "#/default", + default: literal, + }); + expect(schema.default).toEqual(literal); + expect(schema).not.toHaveProperty("$ref"); + }); +}); diff --git a/src/contracts/portablePattern.ts b/src/contracts/portablePattern.ts new file mode 100644 index 0000000000..372c73173b --- /dev/null +++ b/src/contracts/portablePattern.ts @@ -0,0 +1,527 @@ +import { portablePatternViolations } from "./advertisedDialect.js"; +import { + INVALID_BASE64_CHARACTER_PATTERN, + INVALID_BASE64_PADDING_PATTERN, + INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN, +} from "../domain/stringPatterns.js"; + +/** + * An advertised `pattern` re-emitted into the portable regex grammar. + * `exact` matches the same strings as the source under ECMAScript `u` + * semantics, `wider` matches a superset, and `unexpressible` has neither form + * (a backreference, for example), so the advertisement must omit it. Server + * validation keeps the canonical expression either way. + * + * V8's plain, `u` and `v` modes accept the same BMP strings. Known + * limitation: V8 without `u` matches UTF-16 units, so where a wildcard, + * shorthand or negated class must match one astral character (`^.$`, + * `^[^]$`, `^[\S]$`, `^\S\W\D$`), a client compiling without `u` rejects an + * astral character at that position, which the canonical pattern accepts. These + * are rare and keeping the constraint beats dropping it. Literal and class + * astral members are omitted instead (see `#literal`). Repeated classes are + * unaffected, and valid base64, which `^(?:....)*$` checks, is ASCII. + */ +export type PortablePatternProjection = + | { readonly kind: "exact"; readonly pattern: string } + | { readonly kind: "wider"; readonly pattern: string } + | { readonly kind: "unexpressible" }; + +class Unexpressible extends Error {} + +// Any string, as runs separated by NUL. +const ANY_STRING = "[^\\x00]*(?:\\x00[^\\x00]*)*"; +const UNBOUNDED_BASE64_PROBES = new Set([ + INVALID_BASE64_CHARACTER_PATTERN.source, + INVALID_BASE64_PADDING_PATTERN.source, + INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN.source, +]); +const ANY_CHARACTER = "(?:[^\\x00]|\\x00)"; +const NON_ASCII = "[^\\x00-\\x7f]"; +const ALNUM = /^[A-Za-z0-9]$/u; +const LITERAL_OUTSIDE_CLASS = new Set("_ !#%&,:;<=>@`~'\"-/"); +const ESCAPED_OUTSIDE_CLASS = new Set("^$\\.*+?()[]{}|/"); +const LITERAL_IN_CLASS = new Set("!#%&,:;<=>@`~'\" _.+*?$"); +const ESCAPED_IN_CLASS = new Set("^$\\.*+?()[]{}|/-"); +const MAX_REPEAT = 1000; + +const hex = (code: number) => `\\x${code.toString(16).padStart(2, "0")}`; + +const emitOutside = (code: number): string => { + const character = String.fromCodePoint(code); + if (ALNUM.test(character) || LITERAL_OUTSIDE_CLASS.has(character)) + return character; + if (ESCAPED_OUTSIDE_CLASS.has(character)) return `\\${character}`; + return hex(code); +}; + +const emitInClass = (code: number, rangeEndpoint: boolean): string => { + const character = String.fromCodePoint(code); + if (ALNUM.test(character)) return character; + // A punctuator that starts a range is read as a literal followed by `-` in + // some engines, so range endpoints use the hexadecimal form. + if (!rangeEndpoint && LITERAL_IN_CLASS.has(character)) return character; + if (!rangeEndpoint && character !== "/" && ESCAPED_IN_CLASS.has(character)) + return `\\${character}`; + return hex(code); +}; + +/** ASCII members as merged ranges, emitted without repeated punctuators. */ +const emitAsciiMembers = (members: ReadonlySet): string => { + const sorted = [...members].sort((left, right) => left - right); + let emitted = ""; + for (let index = 0; index < sorted.length;) { + let end = index; + while (end + 1 < sorted.length && sorted[end + 1] === sorted[end]! + 1) + end++; + const first = sorted[index]!; + const last = sorted[end]!; + if (last - first >= 2) + emitted += `${emitInClass(first, true)}-${emitInClass(last, true)}`; + else + for (let code = first; code <= last; code++) + emitted += emitInClass(code, false); + index = end + 1; + } + return emitted; +}; + +/** Non-ASCII membership of a class, as far as it changes the projection. */ +type NonAscii = "none" | "some" | "all"; + +const unionNonAscii = (left: NonAscii, right: NonAscii): NonAscii => + left === "all" || right === "all" + ? "all" + : left === "some" || right === "some" + ? "some" + : "none"; + +const ASCII_CODES = Array.from({ length: 0x80 }, (_, code) => code); + +/** + * Re-emit one character class, given its exact ASCII members (computed with + * V8 itself) and how much of the non-ASCII range it contains. + */ +const emitClass = ( + ascii: ReadonlySet, + nonAscii: NonAscii, + negated: boolean, + widen: () => void, +): string => { + if (nonAscii === "some") widen(); + if (!negated) { + if (nonAscii === "none") { + if (ascii.size === 0) throw new Unexpressible(); + return `[${emitAsciiMembers(ascii)}]`; + } + if (ascii.size === 0) return NON_ASCII; + // One negated class admits every non-ASCII character; an alternation of + // two classes would cost a backtracking frame per repeated character. + const missing = new Set(ASCII_CODES.filter((code) => !ascii.has(code))); + return missing.size === 0 + ? ANY_CHARACTER + : `[^${emitAsciiMembers(missing)}]`; + } + if (nonAscii === "all") { + const rest = new Set(ASCII_CODES.filter((code) => !ascii.has(code))); + if (rest.size === 0) throw new Unexpressible(); + return `[${emitAsciiMembers(rest)}]`; + } + // A negated class keeps every non-ASCII character it does not name; with + // "some" named, admitting all of them only widens. + return ascii.size === 0 ? ANY_CHARACTER : `[^${emitAsciiMembers(ascii)}]`; +}; + +const SHORTHAND_NON_ASCII: Readonly> = { + d: "none", + w: "none", + s: "some", + D: "all", + W: "all", + S: "some", +}; + +class Projector { + #index = 0; + #wider = false; + + constructor(private readonly source: string) {} + + project(): { + readonly kind: "exact" | "wider"; + readonly pattern: string; + } { + const alternatives = this.#disjunction(); + if (this.#index !== this.source.length) throw new Unexpressible(); + // A search for an unanchored alternative is a full match of the same + // alternative between two arbitrary strings. + const completed = alternatives.map((terms) => { + const first = terms[0] === "^" ? 1 : 0; + const anchoredEnd = terms.length > first && terms.at(-1) === "$"; + const body = terms.slice(first, anchoredEnd ? -1 : undefined).join(""); + return `${first === 1 ? "" : ANY_STRING}${body}${anchoredEnd ? "" : ANY_STRING}`; + }); + const pattern = + completed.length === 1 + ? `^${completed[0]!}$` + : `^(?:${completed.join("|")})$`; + return { kind: this.#wider ? "wider" : "exact", pattern }; + } + + #peek(offset = 0): string { + return this.source[this.#index + offset] ?? ""; + } + + #widen = (): void => { + this.#wider = true; + }; + + #disjunction(): string[][] { + const alternatives = [this.#alternative()]; + while (this.#peek() === "|") { + this.#index++; + alternatives.push(this.#alternative()); + } + return alternatives; + } + + #group(): string { + return this.#disjunction() + .map((terms) => terms.join("")) + .join("|"); + } + + #alternative(): string[] { + const terms: string[] = []; + while (this.#index < this.source.length) { + const character = this.#peek(); + if (character === "|" || character === ")") break; + const atom = this.#term(); + if (atom === undefined) continue; + const quantifier = this.#quantifier(); + if (atom.kind === "assertion") { + // A quantified assertion matches the same positions or none. + if (quantifier !== "") throw new Unexpressible(); + terms.push(atom.text); + } else + terms.push( + atom.text === ANY_CHARACTER && quantifier === "*" + ? ANY_STRING + : atom.text + quantifier, + ); + } + return terms; + } + + #term(): + | { readonly kind: "atom" | "assertion"; readonly text: string } + | undefined { + const character = this.#peek(); + if (character === "^" || character === "$") { + this.#index++; + return { kind: "assertion", text: character }; + } + if (character === "(") return this.#parenthesized(); + if (character === "[") return { kind: "atom", text: this.#class() }; + if (character === ".") { + this.#index++; + // Without the `s` flag, `.` excludes four line terminators; two are ASCII. + return { + kind: "atom", + text: emitClass(new Set([0x0a, 0x0d]), "some", true, this.#widen), + }; + } + if (character === "\\") return this.#atomEscape(); + const code = this.source.codePointAt(this.#index)!; + this.#index += String.fromCodePoint(code).length; + return { kind: "atom", text: this.#literal(code) }; + } + + #literal(code: number): string { + if (code < 0x80) return emitOutside(code); + // V8 without `u` reads an astral literal as two UTF-16 units, so one + // non-ASCII unit would reject it there, and a one-or-two-unit atom + // backtracks exponentially under a repeat. + if (code > 0xffff) throw new Unexpressible(); + this.#widen(); + return NON_ASCII; + } + + #parenthesized(): + | { readonly kind: "atom" | "assertion"; readonly text: string } + | undefined { + const rest = this.source.slice(this.#index); + const lookAround = /^\(\?/u.exec(rest); + if (named !== null) this.#index += named[0].length; + else if (rest.startsWith("(?")) throw new Unexpressible(); + else this.#index++; + } + const inner = this.#group(); + this.#expect(")"); + return { kind: "atom", text: `${open}${inner})` }; + } + + #expect(character: string): void { + if (this.#peek() !== character) throw new Unexpressible(); + this.#index++; + } + + #quantifier(): string { + const character = this.#peek(); + let quantifier = ""; + if (character === "*" || character === "+" || character === "?") { + this.#index++; + quantifier = character; + } else if (character === "{") { + const repeat = /^\{(\d+)(?:(,)(\d*))?\}/u.exec( + this.source.slice(this.#index), + ); + if (repeat === null) throw new Unexpressible(); + this.#index += repeat[0].length; + let lower = Number(repeat[1]); + let upper = + repeat[2] === undefined + ? lower + : repeat[3] === "" + ? undefined + : Number(repeat[3]); + if (lower > MAX_REPEAT) { + lower = MAX_REPEAT; + upper = undefined; + this.#widen(); + } + if (upper !== undefined && upper > MAX_REPEAT) { + upper = undefined; + this.#widen(); + } + quantifier = + upper === lower + ? `{${String(lower)}}` + : `{${String(lower)},${upper === undefined ? "" : String(upper)}}`; + } else return ""; + // Laziness changes which match is found, not whether one exists. + if (this.#peek() === "?") this.#index++; + return quantifier; + } + + /** One escaped code point, or a shorthand class letter. */ + #escape(inClass: boolean): number | { readonly shorthand: string } { + this.#index++; // backslash + const character = this.#peek(); + this.#index++; + if (character in SHORTHAND_NON_ASCII) return { shorthand: character }; + if (character === "p" || character === "P") { + const property = /^\{[^}]*\}/u.exec(this.source.slice(this.#index)); + if (property === null) throw new Unexpressible(); + this.#index += property[0].length; + return { shorthand: `${character}${property[0]}` }; + } + const controls: Readonly> = { + t: 0x09, + n: 0x0a, + v: 0x0b, + f: 0x0c, + r: 0x0d, + }; + if (Object.hasOwn(controls, character)) return controls[character]!; + if (character === "0" && !/^[0-9]$/u.test(this.#peek())) return 0; + if (character === "b" && inClass) return 0x08; + if (character === "-" && inClass) return 0x2d; + if (character === "c") { + const letter = this.#peek(); + if (!/^[A-Za-z]$/u.test(letter)) throw new Unexpressible(); + this.#index++; + return letter.charCodeAt(0) % 32; + } + if (character === "x") { + const digits = /^[0-9A-Fa-f]{2}/u.exec(this.source.slice(this.#index)); + if (digits === null) throw new Unexpressible(); + this.#index += 2; + return Number.parseInt(digits[0], 16); + } + if (character === "u") { + const braced = /^\{([0-9A-Fa-f]+)\}/u.exec( + this.source.slice(this.#index), + ); + if (braced !== null) { + this.#index += braced[0].length; + return Number.parseInt(braced[1]!, 16); + } + const digits = /^[0-9A-Fa-f]{4}/u.exec(this.source.slice(this.#index)); + if (digits === null) throw new Unexpressible(); + this.#index += 4; + const unit = Number.parseInt(digits[0], 16); + const low = /^\\u([Dd][C-Fc-f][0-9A-Fa-f]{2})/u.exec( + this.source.slice(this.#index), + ); + if (unit >= 0xd800 && unit <= 0xdbff && low !== null) { + this.#index += low[0].length; + return ( + (unit - 0xd800) * 0x400 + + (Number.parseInt(low[1]!, 16) - 0xdc00) + + 0x10000 + ); + } + return unit; + } + // Identity escapes of syntax characters; `u` mode admits no others. + if (ESCAPED_IN_CLASS.has(character) && character !== "-") { + return character.charCodeAt(0); + } + // Backreferences and word boundaries are handled by the caller. + throw new Unexpressible(); + } + + #atomEscape(): + | { readonly kind: "atom" | "assertion"; readonly text: string } + | undefined { + const next = this.#peek(1); + if (next === "b" || next === "B") { + // Removing a zero-width assertion admits a superset of strings. + this.#index += 2; + this.#widen(); + this.#quantifier(); + return undefined; + } + // Backreferences have no portable form. + if (/^[1-9k]$/u.test(next)) throw new Unexpressible(); + const escaped = this.#escape(false); + if (typeof escaped === "number") + return { kind: "atom", text: this.#literal(escaped) }; + const { ascii, nonAscii } = this.#membership(`[\\${escaped.shorthand}]`, [ + escaped, + ]); + return { + kind: "atom", + text: emitClass(ascii, nonAscii, false, this.#widen), + }; + } + + /** Parse a class at the cursor and re-emit it. */ + #class(): string { + const start = this.#index; + this.#index++; + const negated = this.#peek() === "^"; + if (negated) this.#index++; + const members: (number | { shorthand: string } | [number, number])[] = []; + while (this.#peek() !== "]") { + if (this.#index >= this.source.length) throw new Unexpressible(); + const low = this.#classAtom(); + if ( + typeof low === "number" && + this.#peek() === "-" && + this.#peek(1) !== "]" && + this.#peek(1) !== "" + ) { + this.#index++; + const high = this.#classAtom(); + if (typeof high !== "number") throw new Unexpressible(); + members.push([low, high]); + } else members.push(low); + } + this.#index++; + // As for literals: V8 without `u` reads an astral member as two UTF-16 + // units, and the one-unit non-ASCII class emitted for it would reject it. + if ( + members.some((member) => + typeof member === "number" + ? member > 0xffff + : Array.isArray(member) && member[1] > 0xffff, + ) + ) + throw new Unexpressible(); + const source = this.source.slice(start, this.#index); + const { ascii: matched, nonAscii } = this.#membership(source, members); + const ascii = negated + ? new Set(ASCII_CODES.filter((code) => !matched.has(code))) + : matched; + return emitClass(ascii, nonAscii, negated, this.#widen); + } + + #classAtom(): number | { shorthand: string } { + if (this.#peek() === "\\") return this.#escape(true); + const code = this.source.codePointAt(this.#index)!; + this.#index += String.fromCodePoint(code).length; + return code; + } + + #membership( + classSource: string, + members: readonly (number | { shorthand: string } | [number, number])[], + ): { readonly ascii: ReadonlySet; readonly nonAscii: NonAscii } { + // V8 decides ASCII membership exactly, including property escapes. + const expression = new RegExp(`^${classSource}$`, "u"); + const ascii = new Set( + ASCII_CODES.filter((code) => expression.test(String.fromCharCode(code))), + ); + let nonAscii: NonAscii = "none"; + for (const member of members) { + if (typeof member === "number") + nonAscii = unionNonAscii(nonAscii, member >= 0x80 ? "some" : "none"); + else if (Array.isArray(member)) + nonAscii = unionNonAscii( + nonAscii, + member[1] < 0x80 + ? "none" + : member[0] <= 0x80 && member[1] >= 0x10ffff + ? "all" + : "some", + ); + else + nonAscii = unionNonAscii( + nonAscii, + SHORTHAND_NON_ASCII[member.shorthand] ?? "some", + ); + } + return { ascii, nonAscii }; + } +} + +const projections = new Map(); + +/** + * Re-emit an advertised ECMAScript pattern in the portable grammar. Results + * are memoized: advertised patterns come from a fixed set of contracts. + */ +export const projectPortablePattern = ( + pattern: string, +): PortablePatternProjection => { + const cached = projections.get(pattern); + if (cached !== undefined) return cached; + let projection: PortablePatternProjection; + try { + // Anchoring these unbounded negative probes requires a repeated universal + // prefix; V8 v-mode can overflow its stack on a large valid base64 value. + if (UNBOUNDED_BASE64_PROBES.has(pattern)) throw new Unexpressible(); + // JSON Schema validators read `pattern` with Unicode semantics. + new RegExp(pattern, "u"); + const projected = new Projector(pattern).project(); + // The emitter's own output must stay inside the grammar it targets. + projection = + portablePatternViolations(projected.pattern).length > 0 + ? { kind: "unexpressible" } + : projected; + } catch (cause: unknown) { + if (!(cause instanceof Unexpressible || cause instanceof SyntaxError)) + throw cause; + projection = { kind: "unexpressible" }; + } + projections.set(pattern, projection); + return projection; +}; diff --git a/src/contracts/toolSchemaMetadata.ts b/src/contracts/toolSchemaMetadata.ts index 27344a19d0..17c007056c 100644 --- a/src/contracts/toolSchemaMetadata.ts +++ b/src/contracts/toolSchemaMetadata.ts @@ -6,7 +6,10 @@ import { z } from "zod"; import type { ToolContract } from "./toolContractTypes.js"; import { advertisedInputExamples } from "./advertisedInputExamples.js"; -import { presentInputJsonSchema } from "./inputSchemaPresentation.js"; +import { + presentInputJsonSchema, + presentOutputJsonSchema, +} from "./inputSchemaPresentation.js"; import { schemaArrays, schemaChildren, @@ -142,10 +145,12 @@ export const toolOutputSchemaWithMetadata = ( contract.outputSchema, "output", (project) => (options) => { - const shared = project({ - ...options, - libraryOptions: { reused: "ref", ...options.libraryOptions }, - }); + const shared = presentOutputJsonSchema( + project({ + ...options, + libraryOptions: { reused: "ref", ...options.libraryOptions }, + }), + ); // An explicit ID owns the reference base. Relative external references // also depend on that base, so leave those projections unchanged. if ("$id" in shared || hasRelativeSchemaReference(shared)) diff --git a/src/server/JavaScriptRouteAdvertisedSchema.test.ts b/src/server/JavaScriptRouteAdvertisedSchema.test.ts index 39215371e2..af30e01a66 100644 --- a/src/server/JavaScriptRouteAdvertisedSchema.test.ts +++ b/src/server/JavaScriptRouteAdvertisedSchema.test.ts @@ -5,6 +5,7 @@ import { createRequire } from "node:module"; import { expect, it } from "vitest"; import { toolContract } from "../contracts/toolContracts.js"; +import { projectPortablePattern } from "../contracts/portablePattern.js"; import { toolOutputSchemaWithMetadata } from "../contracts/toolSchemaMetadata.js"; import { javaScriptRouteCandidateSchema } from "../domain/javascript/javascriptRouteFacts.js"; import { javaScriptInventoryRouteSchema } from "../domain/javascript/javascriptRouteInventory.js"; @@ -47,7 +48,10 @@ it("advertises JavaScript route outputs with standard JSON Schema path constrain addFormats(ajv); expect(() => ajv.compile(advertised as AnySchema)).not.toThrow(); - // The inventory path is nested inside the advertised complete Evidence branch. + // The inventory path is nested inside the advertised complete Evidence + // branch, re-emitted as a portable anchored pattern. + const portable = projectPortablePattern("^\\/"); + expect(portable.kind).toBe("exact"); const patterns: Record[] = []; const visit = (value: unknown): void => { if (value === null || typeof value !== "object") return; @@ -56,7 +60,11 @@ it("advertises JavaScript route outputs with standard JSON Schema path constrain return; } const record = value as Record; - if (record.pattern === "^\\/") patterns.push(record); + if ( + portable.kind !== "unexpressible" && + record.pattern === portable.pattern + ) + patterns.push(record); Object.values(record).forEach(visit); }; visit(advertised); diff --git a/tests/boundary/mcp/advertisedDialect.test.ts b/tests/boundary/mcp/advertisedDialect.test.ts index 8b735905f0..b3e2dae1ca 100644 --- a/tests/boundary/mcp/advertisedDialect.test.ts +++ b/tests/boundary/mcp/advertisedDialect.test.ts @@ -10,10 +10,7 @@ import { portablePatternViolations, type AdvertisedToolDefinition, } from "../../../src/contracts/advertisedDialect.js"; -import { - ADVERTISED_INPUT_DIALECT_BASELINE, - ADVERTISED_OUTPUT_DIALECT_BASELINE, -} from "../../../src/contracts/advertisedDialectBaseline.js"; +import { ADVERTISED_INPUT_DIALECT_BASELINE } from "../../../src/contracts/advertisedDialectBaseline.js"; import { mcpInputSchemaProfileSchema } from "../../../src/config/mcpInputSchemaProfile.js"; import { mcpOutputSchemaProfileSchema } from "../../../src/config/mcpOutputSchemaProfile.js"; import { mcpToolCatalogProfileSchema } from "../../../src/config/mcpToolCatalogProfile.js"; @@ -98,7 +95,7 @@ const distinct = ( it("advertises every profile within the portable dialect or its recorded baseline", async () => { const inputs = new Map>(); - const outputs = new Map>(); + const outputViolations: string[] = []; const uncompilable: string[] = []; for (const input of mcpInputSchemaProfileSchema.options) for (const output of mcpOutputSchemaProfileSchema.options) @@ -120,19 +117,18 @@ it("advertises every profile within the portable dialect or its recorded baselin ).toBeGreaterThan(0); for (const tool of tools) { if (!inputs.has(input)) inputs.set(input, new Map()); - if (!outputs.has(output)) outputs.set(output, new Map()); distinct(inputs.get(input)!, { name: tool.name, description: tool.description, inputSchema: tool.inputSchema, }); - distinct(outputs.get(output)!, { - name: tool.name, - inputSchema: {}, - outputSchema: tool.outputSchema, - }); + // SDK clients compile every listed output schema. + for (const violation of advertisedOutputPatternViolations(tool)) + outputViolations.push( + `${combination} ${tool.name} ${violation.path} ${violation.detail ?? ""}`, + ); // The grammar was checked against seven engines offline; this keeps - // V8's own modes, including baselined patterns, checked in CI. + // V8's own modes checked in CI. for (const schema of [tool.inputSchema, tool.outputSchema]) for (const { path, pattern } of advertisedSchemaPatterns(schema)) for (const flags of ["", "u", "v"]) @@ -146,30 +142,21 @@ it("advertises every profile within the portable dialect or its recorded baselin } } expect(uncompilable).toEqual([]); + expect(outputViolations).toEqual([]); - const observed = { - input: Object.fromEntries( - [...inputs].map(([profile, tools]) => [ - profile, - countAdvertisedDialectViolations( - [...tools.values()].flatMap(advertisedInputDialectViolations), - ), - ]), - ), - output: Object.fromEntries( - [...outputs].map(([profile, tools]) => [ - profile, - countAdvertisedDialectViolations( - [...tools.values()].flatMap(advertisedOutputPatternViolations), - ), - ]), - ), - }; + const observed = Object.fromEntries( + [...inputs].map(([profile, tools]) => [ + profile, + countAdvertisedDialectViolations( + [...tools.values()].flatMap(advertisedInputDialectViolations), + ), + ]), + ); // New constructs fail first with a named key; a count that dropped below // the baseline must be recorded so it cannot grow back. - expect({ - input: Object.fromEntries( - Object.entries(observed.input).map(([profile, counts]) => [ + expect( + Object.fromEntries( + Object.entries(observed).map(([profile, counts]) => [ profile, advertisedDialectRegressions( counts, @@ -179,28 +166,6 @@ it("advertises every profile within the portable dialect or its recorded baselin ), ]), ), - output: Object.fromEntries( - Object.entries(observed.output).map(([profile, counts]) => [ - profile, - advertisedDialectRegressions( - counts, - ADVERTISED_OUTPUT_DIALECT_BASELINE[ - profile as keyof typeof ADVERTISED_OUTPUT_DIALECT_BASELINE - ], - ), - ]), - ), - }).toEqual({ - input: { compact: [], full: [] }, - output: { full: [], compact: [], off: [] }, - }); - // Absolute local paths are advertised with the host's pattern - // (`localPath.ts`): POSIX hosts advertise `^\/`, which the grammar counts, - // and Windows hosts a drive-or-root pattern, which it does not. Exact counts - // are recorded on POSIX; Windows still enforces them as upper bounds above. - if (process.platform !== "win32") - expect(observed).toEqual({ - input: ADVERTISED_INPUT_DIALECT_BASELINE, - output: ADVERTISED_OUTPUT_DIALECT_BASELINE, - }); + ).toEqual({ compact: [], full: [] }); + expect(observed).toEqual(ADVERTISED_INPUT_DIALECT_BASELINE); }); diff --git a/tests/boundary/mcp/advertisedPatternsVMode.test.ts b/tests/boundary/mcp/advertisedPatternsVMode.test.ts index 29ea3a5e36..7ba9109e94 100644 --- a/tests/boundary/mcp/advertisedPatternsVMode.test.ts +++ b/tests/boundary/mcp/advertisedPatternsVMode.test.ts @@ -4,6 +4,7 @@ import { expect, it, onTestFinished } from "vitest"; import { createServer } from "../../../src/server/createServer.js"; import { advertisedSchemaPatterns } from "../../../src/contracts/advertisedDialect.js"; +import { projectPortablePattern } from "../../../src/contracts/portablePattern.js"; import { BASE64_QUARTET_LENGTH_PATTERN, INVALID_BASE64_CHARACTER_PATTERN, @@ -14,7 +15,14 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; // Pattern portability across every profile is owned by // advertisedDialect.test.ts. This file keeps the composed canonical base64 -// constraint's advertised semantics. +// constraint's safe advertisement after portable re-emission. + +const advertised = (pattern: RegExp): string => { + const projection = projectPortablePattern(pattern.source); + if (projection.kind === "unexpressible") + throw new Error(`No portable form for ${pattern.source}`); + return projection.pattern; +}; const collectCanonicalBase64Leaves = ( value: unknown, @@ -34,7 +42,7 @@ const collectCanonicalBase64Leaves = ( entry !== null && typeof entry === "object" && "pattern" in entry && - entry.pattern === BASE64_QUARTET_LENGTH_PATTERN.source, + entry.pattern === advertised(BASE64_QUARTET_LENGTH_PATTERN), ) ) leaves.push(record); @@ -61,11 +69,11 @@ const listAdvertisedTools = async () => { return (await client.listTools()).tools; }; -it("advertises the composed canonical base64 constraint with exact SDK semantics", async () => { - const advertised = await listAdvertisedTools(); +it("advertises base64 without rejecting valid inputs or overflowing V8", async () => { + const tools = await listAdvertisedTools(); const patterns: string[] = []; const canonicalBase64Leaves: Record[] = []; - for (const tool of advertised) + for (const tool of tools) for (const schema of [tool.inputSchema, tool.outputSchema]) { if (schema === undefined) continue; patterns.push( @@ -74,31 +82,32 @@ it("advertises the composed canonical base64 constraint with exact SDK semantics collectCanonicalBase64Leaves(schema, canonicalBase64Leaves); } + expect(patterns).toContain(advertised(BASE64_QUARTET_LENGTH_PATTERN)); for (const pattern of [ - BASE64_QUARTET_LENGTH_PATTERN, INVALID_BASE64_CHARACTER_PATTERN, INVALID_BASE64_PADDING_PATTERN, INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN, ]) - expect( - patterns.some((advertisedPattern) => - advertisedPattern.includes(pattern.source), - ), - pattern.source, - ).toBe(true); + expect(projectPortablePattern(pattern.source)).toEqual({ + kind: "unexpressible", + }); expect(canonicalBase64Leaves.length).toBeGreaterThan(0); const sdkValidator = new AjvJsonSchemaValidator(); const validators = canonicalBase64Leaves.map((schema) => sdkValidator.getValidator(schema), ); - for (const value of ["", "AA==", "AQID", "AB==", "AAA", "AA$=", "AA\n=="]) + for (const value of ["", "AA==", "AQID"]) expect( validators.map((validate) => validate(value).valid), JSON.stringify(value), - ).toEqual( - canonicalBase64Leaves.map( - () => value === "" || value === "AA==" || value === "AQID", - ), - ); + ).toEqual(canonicalBase64Leaves.map(() => true)); + const large = "A".repeat(8 * 1024 * 1024); + const base64Patterns = new Set( + canonicalBase64Leaves.flatMap((schema) => + advertisedSchemaPatterns(schema).map(({ pattern }) => pattern), + ), + ); + for (const pattern of base64Patterns) + expect(new RegExp(pattern, "v").test(large)).toBe(true); });