From 5e170e4886cf2fe3262e9445850d57e38430d629 Mon Sep 17 00:00:00 2001
From: haelyra <49814733+haelyra@users.noreply.github.com>
Date: Thu, 1 Oct 2026 12:04:10 -0400
Subject: [PATCH] fix(bengali): preserve decimal tokens

---
 pi/core/skills/bengali-nlp/SKILL.md       |  4 ++--
 skills/bengali-nlp/SKILL.md               |  4 ++--
 tests/ci/bengali-language-surface.test.js | 26 +++++++++++++++++++++++
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/pi/core/skills/bengali-nlp/SKILL.md b/pi/core/skills/bengali-nlp/SKILL.md
index c16b41946..74a81067f 100644
--- a/pi/core/skills/bengali-nlp/SKILL.md
+++ b/pi/core/skills/bengali-nlp/SKILL.md
@@ -131,8 +131,8 @@ import re
 
 def tokenize_bangla(text: str) -> list[str]:
     """Basic Bengali tokenizer — split on whitespace and punctuation."""
-    # Remove Bengali and ASCII punctuation
-    text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)
+    # Remove Bengali and ASCII punctuation, but preserve decimal points
+    text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)
     return [token for token in text.split() if token]
 
 # For production, use a trained tokenizer:
diff --git a/skills/bengali-nlp/SKILL.md b/skills/bengali-nlp/SKILL.md
index c16b41946..74a81067f 100644
--- a/skills/bengali-nlp/SKILL.md
+++ b/skills/bengali-nlp/SKILL.md
@@ -131,8 +131,8 @@ import re
 
 def tokenize_bangla(text: str) -> list[str]:
     """Basic Bengali tokenizer — split on whitespace and punctuation."""
-    # Remove Bengali and ASCII punctuation
-    text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)
+    # Remove Bengali and ASCII punctuation, but preserve decimal points
+    text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)
     return [token for token in text.split() if token]
 
 # For production, use a trained tokenizer:
diff --git a/tests/ci/bengali-language-surface.test.js b/tests/ci/bengali-language-surface.test.js
index cb88d3153..06e3c1955 100644
--- a/tests/ci/bengali-language-surface.test.js
+++ b/tests/ci/bengali-language-surface.test.js
@@ -143,6 +143,32 @@ function run() {
     });
   });
 
+  test('Bengali tokenizer preserves decimal quantities', () => {
+    const skill = read('skills/bengali-nlp/SKILL.md');
+    const tokenizer = extractFencedBlocks(skill)
+      .filter(block => block.language === 'python')
+      .find(block => block.code.includes('def tokenize_bangla'));
+    assert.ok(tokenizer, 'Expected the Bengali tokenizer example');
+
+    const python = findPython();
+    if (!python) {
+      throw new SkipError('no Python 3 interpreter on PATH');
+    }
+
+    const check = [
+      tokenizer.code,
+      "assert tokenize_bangla('ওজন ৩.৫ কেজি। মূল্য 12.75 টাকা.') == ['ওজন', '৩.৫', 'কেজি', 'মূল্য', '12.75', 'টাকা']",
+    ].join('\n');
+    const result = spawnSync(python.command, [...python.prefix, '-c', check], {
+      encoding: 'utf8',
+    });
+    assert.strictEqual(
+      result.status,
+      0,
+      `Bengali tokenizer changed decimal quantities:\n${result.stderr || result.stdout}`,
+    );
+  });
+
   test('Bengali skill is exported and installable', () => {
     const agentYaml = read('agent.yaml');
     const modules = readJson('manifests/install-modules.json').modules;