mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 11:43:20 +00:00
fix(environment): preserve raw UTF-16 token boundaries
Pair a preceding low surrogate only with its adjacent high surrogate. Unpaired code units must not make an earlier letter become the token's neighbor. Keep UTF-16 offsets and source strings unchanged at the shared delimiter owner used by rule discovery, copy checks, and replacement. Cover valid astral letters, digits, emoji, and lone-surrogate controls, plus create/read behavior through a CLI-started environment MCP server. Validation: regression-first run had 4 expected failures; after the fix, 437 selected domain, application, and server-boundary tests passed.
This commit is contained in:
1 parent
8681752c7b
commit
54646e8dc4
3 files changed
+101
-4
No files matched your search
@@ -266,9 +266,11 @@ const isAlphanumeric = (character: string | undefined): boolean =>
|
||||
const previousCharacter = (text: string, at: number): string | undefined => {
|
||||
if (at === 0) return undefined;
|
||||
const last = text.charCodeAt(at - 1);
|
||||
return last >= 0xdc00 && last <= 0xdfff && at > 1
|
||||
? text.slice(at - 2, at)
|
||||
: text[at - 1];
|
||||
if (last >= 0xdc00 && last <= 0xdfff && at > 1) {
|
||||
const first = text.charCodeAt(at - 2);
|
||||
if (first >= 0xd800 && first <= 0xdbff) return text.slice(at - 2, at);
|
||||
}
|
||||
return text[at - 1];
|
||||
};
|
||||
|
||||
const nextCharacter = (text: string, at: number): string | undefined => {
|
||||
|
||||
Reference in new issue
Block a user