fix(environment): preserve raw UTF-16 token boundaries

Pair a preceding low surrogate only with its adjacent high surrogate.
Unpaired code units must not make an earlier letter become the token's
neighbor. Keep UTF-16 offsets and source strings unchanged at the shared
delimiter owner used by rule discovery, copy checks, and replacement.

Cover valid astral letters, digits, emoji, and lone-surrogate controls,
plus create/read behavior through a CLI-started environment MCP server.
Validation: regression-first run had 4 expected failures; after the fix,
437 selected domain, application, and server-boundary tests passed.
This commit is contained in:
morluto committed 2026-10-24 08:58:22 +08:00
1 parent 8681752c7b
commit 54646e8dc4
3 files changed
+101 -4

No files matched your search

@@ -266,9 +266,11 @@ const isAlphanumeric = (character: string | undefined): boolean =>
const previousCharacter = (text: string, at: number): string | undefined => {
if (at === 0) return undefined;
const last = text.charCodeAt(at - 1);
return last >= 0xdc00 && last <= 0xdfff && at > 1
? text.slice(at - 2, at)
: text[at - 1];
if (last >= 0xdc00 && last <= 0xdfff && at > 1) {
const first = text.charCodeAt(at - 2);
if (first >= 0xd800 && first <= 0xdbff) return text.slice(at - 2, at);
}
return text[at - 1];
};
const nextCharacter = (text: string, at: number): string | undefined => {