diff --git a/agents/bengali-reviewer.md b/agents/bengali-reviewer.md index b46213d8e..a6682fcf2 100644 --- a/agents/bengali-reviewer.md +++ b/agents/bengali-reviewer.md @@ -117,8 +117,12 @@ first_char = graphemes[0] # Returns 'ক্ষ' as a complete unit from Unicode code point order. - **Tokenization splitting on whitespace only** — Bengali has compound words and postposition attachments that require linguistic tokenization. -- **Regex not covering Bengali range** — Character class patterns like `\w` or `[a-zA-Z]` - that exclude Bengali characters when Bengali input is expected. +- **Regex not covering Bengali range** — Character classes like `[a-zA-Z]` that + exclude Bengali characters when Bengali input is expected, and `\w` used to match + Bengali words. `\w` is ASCII-only in JavaScript (with or without the `u` flag) and + in Python with `re.ASCII`. In Python `str` patterns it matches Bengali letters but + not vowel signs or hasanta (category M), so `re.findall(r'\w+', 'রহিম')` returns + `['রহ', 'ম']`. Recommend `[ঀ-৿]` or the `regex` module's `\p{Bengali}`. - **Bengali digits not handled** — Validation or parsing that rejects Bengali digits (০-৯) when users may input them. diff --git a/skills/bengali-nlp/SKILL.md b/skills/bengali-nlp/SKILL.md index 305fd9fdd..e5dabca16 100644 --- a/skills/bengali-nlp/SKILL.md +++ b/skills/bengali-nlp/SKILL.md @@ -339,12 +339,16 @@ CREATE TABLE bengali_content ( -- FROM pg_database WHERE datname = current_database(); -- -- 2. The exact collation must exist with the right provider and locale. --- `collcollate` holds the locale on PG < 15; `colllocale` holds it on PG 15+, --- so check both and require a non-null match: --- SELECT collprovider = 'i' AS icu_provider, --- coalesce(colllocale, collcollate) AS locale --- FROM pg_collation --- WHERE collname = 'bn-BD-x-icu'; +-- The ICU locale column moved between releases: `collcollate` on PG <= 14, +-- `colliculocale` on PG 15-16, `colllocale` on PG 17+. Naming a column the +-- server lacks is an error, so read them through to_jsonb(), which works on +-- every version, and require a non-null match: +-- SELECT c.collprovider = 'i' AS icu_provider, +-- coalesce(to_jsonb(c) ->> 'colllocale', +-- to_jsonb(c) ->> 'colliculocale', +-- c.collcollate) AS locale +-- FROM pg_collation c +-- WHERE c.collname = 'bn-BD-x-icu'; -- -- A name-only `CREATE COLLATION IF NOT EXISTS` is not enough: it silently keeps -- an existing collation that may be bound to a different provider or locale.