fix(bengali): preserve decimal tokens

pi/core/skills/bengali-nlp/SKILL.md

@@ -131,8 +131,8 @@ import re
 
 def tokenize_bangla(text: str) -> list[str]:
     """Basic Bengali tokenizer — split on whitespace and punctuation."""
-    # Remove Bengali and ASCII punctuation
-    text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)
+    # Remove Bengali and ASCII punctuation, but preserve decimal points
+    text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)
     return [token for token in text.split() if token]
 
 # For production, use a trained tokenizer:

skills/bengali-nlp/SKILL.md

@@ -131,8 +131,8 @@ import re
 
 def tokenize_bangla(text: str) -> list[str]:
     """Basic Bengali tokenizer — split on whitespace and punctuation."""
-    # Remove Bengali and ASCII punctuation
-    text = re.sub(r'[।,;:!?.\-\'"()\[\]{}]', ' ', text)
+    # Remove Bengali and ASCII punctuation, but preserve decimal points
+    text = re.sub(r'(?<!\d)\.|\.(?!\d)|[।,;:!?\-\'"()\[\]{}]', ' ', text)
     return [token for token in text.split() if token]
 
 # For production, use a trained tokenizer:

tests/ci/bengali-language-surface.test.js

@@ -143,6 +143,32 @@ function run() {
     });
   });
 
+  test('Bengali tokenizer preserves decimal quantities', () => {
+    const skill = read('skills/bengali-nlp/SKILL.md');
+    const tokenizer = extractFencedBlocks(skill)
+      .filter(block => block.language === 'python')
+      .find(block => block.code.includes('def tokenize_bangla'));
+    assert.ok(tokenizer, 'Expected the Bengali tokenizer example');
+
+    const python = findPython();
+    if (!python) {
+      throw new SkipError('no Python 3 interpreter on PATH');
+    }
+
+    const check = [
+      tokenizer.code,
+      "assert tokenize_bangla('ওজন ৩.৫ কেজি। মূল্য 12.75 টাকা.') == ['ওজন', '৩.৫', 'কেজি', 'মূল্য', '12.75', 'টাকা']",
+    ].join('\n');
+    const result = spawnSync(python.command, [...python.prefix, '-c', check], {
+      encoding: 'utf8',
+    });
+    assert.strictEqual(
+      result.status,
+      0,
+      `Bengali tokenizer changed decimal quantities:\n${result.stderr || result.stdout}`,
+    );
+  });
+
   test('Bengali skill is exported and installable', () => {
     const agentYaml = read('agent.yaml');
     const modules = readJson('manifests/install-modules.json').modules;