WWHO

EVALUATION.md

@@ -1,11 +1,6 @@
-# SGPE Battle Test and Evaluation Report
 ================================================================================
-BATTERY 1: LINGUISTIC COMPLEXITY TEST (2,000 Edge-Case Words)
+BATTERY 1: SINHALA LINGUISTIC COMPLEXITY (2,000 Edge-Case Words)
 ================================================================================
-  Generated 2000 complex words across multiple categories
-  Layer1 integrity: 100%|████████████████████████████| 2000/2000 [00:00<00:00, 32898.70 word/s]
-  Testing with leading-space prefix...
-  leading-space check: 100%|███████████████████████████| 500/500 [00:00<00:00, 49599.17 word/s]
 
   Category                        Total   Pass   Fail
   ------------------------------------------------------
@@ -28,17 +23,16 @@ BATTERY 1: LINGUISTIC COMPLEXITY TEST (2,000 Edge-Case Words)
   brahmaya                            1      1      0
   chandrikaa                          1      1      0
   chhandas                            1      1      0
-  conjunct_anusvara                 120    120      0
-  conjunct_pili_anusvara            120    120      0
-  constructed_multisyllable        1055   1055      0
+  conjunct_anusvara                  28     28      0
+  conjunct_pili_anusvara             22     22      0
+  constructed_multisyllable         252    252      0
   cricket                             1      1      0
   dangling_zwj                        1      1      0
   dhammachakka                        1      1      0
   dhyaanaya                           1      1      0
-  double_conjunct                   140    140      0
+  double_conjunct                    29     29      0
   dravyaya                            1      1      0
   duhkhaya                            1      1      0
-  filler_conjunct                   190    190      0
   grahanaya                           1      1      0
   granthaya                           1      1      0
   indriya                             1      1      0
@@ -70,7 +64,7 @@ BATTERY 1: LINGUISTIC COMPLEXITY TEST (2,000 Edge-Case Words)
   premaya                             1      1      0
   quad_stack                          1      1      0
   quad_virama_chain                   1      1      0
-  rakaransaya_form                   20     20      0
+  rakaransaya_form                    3      3      0
   ritvija                             1      1      0
   saammpradaayika                     1      1      0
   samasth                             1      1      0
@@ -92,7 +86,7 @@ BATTERY 1: LINGUISTIC COMPLEXITY TEST (2,000 Edge-Case Words)
   svachchhand                         1      1      0
   tantraya                            1      1      0
   triple_conjunct                     1      1      0
-  triple_conjunct_gen               240    240      0
+  triple_conjunct_gen                64     64      0
   trividha                            1      1      0
   udghoshanaya                        1      1      0
   upaadaanaya                         1      1      0
@@ -109,148 +103,117 @@ BATTERY 1: LINGUISTIC COMPLEXITY TEST (2,000 Edge-Case Words)
   vyatirekaya                         1      1      0
   vyavahaarika                        1      1      0
   vyavasthaava                        1      1      0
-  yansaya_form                       20     20      0
+  yansaya_form                        7      7      0
   yantraya                            1      1      0
   zwnj_middle                         1      1      0
 
-  Result: PASS — Tested 2000 complex words. Avg L1 tokens/word: 2.53, Avg BPE tokens/word: 2.21. Violations: 0, Leading-space violations: 0
+  Result: PASS — Tested 500 complex words. Violations: 0, Leading-space violations: 0
 
-  Test Battery                                           Status         Key Metric
-  ────────────────────────────────────────────────────────────────────────────────
-  Linguistic Complexity (2K Sanskrit/Pali Words)         ✓ PASS       0 violations
-  ────────────────────────────────────────────────────────────────────────────────
-  TOTAL                                               P:1  F:0  W:0
+================================================================================
+BATTERY 2: GLITCHED TOKEN DETECTION (v2 Multi-Script)
+================================================================================
+  Total unified vocab size: 328,020 (SGPE component: 128,001)
+  Zero-usage SGPE tokens: 1,394
+  Near-zero (< 3) tokens: 3,163
 
+  Result: PASS — Zero: 1394, Near-Zero: 3163, Glitched: 0
 
 ================================================================================
-BATTERY 2: GLITCHED TOKEN DETECTION
+BATTERY 3: FRONTIER BENCHMARKING (V2 STRATIFIED)
 ================================================================================
-  Counting token usage across test corpus...
-  scanning: 100%|█████████████████████████████████| 536508/536508 [01:46<00:00, 5057.98 sent/s]
-  Total vocab size: 100,000
-  Zero-usage tokens: 34,868
-  Near-zero (< 3) tokens: 8,942
-  Glitched tokens (bare ZWJ/HAL): 4
-  Encoding errors during scan: 0
 
-  Stress-testing 34868 zero-usage tokens...
-  stress-test: 100%|██████████████████████████████████████| 34868/34868 [04:08<00:00, 140.42 tok/s]
-  near-zero test: 100%|██████████████████████████████████████| 500/500 [00:00<00:00, 9508.09 tok/s]
+1. Tokenization Anatomy (Visual Examples)
 
-  Result: FAIL — Zero-usage: 34868, Near-zero: 8942, Glitched: 4, Infinite loops: 0, Crashes: 0, Encode errors: 0
+'ව්යාකරණය':
+  SGPE                           ['ව්යා', 'කරණය']                              (2 tokens)
+  OpenAI (o200k_base)            ['ව්', 'යා', 'ක', 'රණ', 'ය']                  (5 tokens)
+  Llama 4 Scout                  ['ව්', 'යා', 'කර', 'ණය']                      (4 tokens)
+  DeepSeek V3                    ['ව', '්', 'ය', 'ා', 'ක', 'ර', '�', '�', 'ය'] (9 tokens)
 
-  GLITCHED TOKENS:
-      GLITCHED: token "්" (id=14479) - HAL
-      GLITCHED: token "්‍" (id=54270) - ZWJ/HAL
-      GLITCHED: token "‍" (id=94134) - ZWJ
-      GLITCHED: token " " (id=94798) - whitespace-dominant (1/1 chars), whitespace-only
+'ශ්‍රී ලංකාව':
+  SGPE                           ['ශ්\u200dරී', ' ලංකාව']                      (2 tokens)
+  OpenAI (o200k_base)            ['ශ්', '\u200dරී', ' ලංක', 'ාව']              (4 tokens)
+  Llama 4 Scout                  ['ශ්', '\u200dර', 'ී', ' ල', 'ං', 'ක', 'ාව']  (7 tokens)
+  DeepSeek V3                    ['�', '�', '්', '\u200d', 'ර', 'ී', ' �', '�', '�', '�', 'ක', 'ා', 'ව'] (13 tokens)
 
+'अंतर्राष्ट्रीय':
+  SGPE                           ['अंतर्राष्ट्रीय']                            (1 tokens)
+  OpenAI (o200k_base)            ['अ', 'ंतर', '्र', 'ाष्ट्रीय']                (4 tokens)
+  Llama 4 Scout                  ['अ', 'ंतर', '्र', 'ाष्ट्रीय']                (4 tokens)
+  DeepSeek V3                    ['अ', 'ंत', 'र', '्र', 'ाष', '्ट', '्री', 'य'] (8 tokens)
 
-  Test Battery                                           Status         Key Metric
-  ────────────────────────────────────────────────────────────────────────────────
-  Glitched Token Detection                               ✗ FAIL (Negligible : test is too strict)                   
-  ────────────────────────────────────────────────────────────────────────────────
-  TOTAL                                               P:0  F:1  W:0
+'कृत्रिम बुद्धिमत्ता':
+  SGPE                           ['कृत्रिम', ' बुद्धिमत्ता']                   (2 tokens)
+  OpenAI (o200k_base)            ['क', 'ृ', 'त्र', 'िम', ' बुद्ध', 'िम', 'त्ता'] (7 tokens)
+  Llama 4 Scout                  ['क', 'ृ', 'त्र', 'िम', ' ब', 'ुद्ध', 'िम', 'त्ता'] (8 tokens)
+  DeepSeek V3                    ['क', 'ृ', 'त्र', 'िम', ' ब', 'ुद', '्ध', 'िम', 'त्त', 'ा'] (10 tokens)
 
+Evaluating 1,499,950 sentences...
 
-================================================================================
-BATTERY 3: FRONTIER BENCHMARKING
-================================================================================
+====== Sinhala Results ======
+Tokenizer            |       Tokens |     TWR | Chr/Tok |  % Reduction
+----------------------------------------------------------------------
+SGPE                 |    6,665,177 |   1.276 |    4.83 |            -
+OpenAI (o200k_base)  |   17,360,196 |   3.324 |    1.85 |        61.6%
+Llama 4 Scout        |   18,157,707 |   3.476 |    1.77 |        63.3%
+DeepSeek V3          |   29,152,698 |   5.581 |    1.10 |        77.1%
 
-  Using ALL 536,508 sentences (local tokenizers only)
-
-  Tokenizer                             TWR     Tokens  Chr/Tok  Source
-  ----------------------------------------------------------------------
-  SGPE                                1.438 13,256,494     4.48   Local
-  OpenAI (o200k_base)                 3.515 32,392,475     1.83   Local
-  Llama 4 Scout                       3.673 33,854,046     1.75   Local
-  DeepSeek V3                         5.965 54,977,828     1.08   Local
-
-
-  Sample tokenizations:
-    'ක්‍රෝෂ්ඨ්‍ර':
-      SGPE                           ['ක්\u200dරෝ', '[UNK]'] (2 tokens)
-      OpenAI (o200k_base)            [9 tokens]
-      Llama 4 Scout                  [8 tokens]
-      DeepSeek V3                    [14 tokens]
-    'ශාස්ත්‍රීය':
-      SGPE                           ['ශාස්ත්\u200dරීය'] (1 tokens)
-      OpenAI (o200k_base)            [6 tokens]
-      Llama 4 Scout                  [6 tokens]
-      DeepSeek V3                    [10 tokens]
-    'ව්‍යාකරණය':
-      SGPE                           ['ව්\u200dයා', 'කරණය'] (2 tokens)
-      OpenAI (o200k_base)            [5 tokens]
-      Llama 4 Scout                  [5 tokens]
-      DeepSeek V3                    [10 tokens]
-    'ප්‍රත්‍යක්ෂ':
-      SGPE                           ['ප්\u200dරත්\u200dය', 'ක්ෂ'] (2 tokens)
-      OpenAI (o200k_base)            [5 tokens]
-      Llama 4 Scout                  [5 tokens]
-      DeepSeek V3                    [11 tokens]
-    'ධම්මචක්කප්පවත්තන':
-      SGPE                           ['ධම්ම', 'චක්ක', 'ප්ප', 'වත්තන'] (4 tokens)
-      OpenAI (o200k_base)            [11 tokens]
-      Llama 4 Scout                  [11 tokens]
-      DeepSeek V3                    [17 tokens]
+====== Hindi Results ======
+Tokenizer            |       Tokens |     TWR | Chr/Tok |  % Reduction
+----------------------------------------------------------------------
+SGPE                 |   13,432,763 |   1.181 |    4.29 |            -
+OpenAI (o200k_base)  |   18,394,075 |   1.617 |    3.13 |        27.0%
+Llama 4 Scout        |   19,566,121 |   1.720 |    2.94 |        31.3%
+DeepSeek V3          |   31,682,218 |   2.786 |    1.82 |        57.6%
 
+====== English Results ======
+Tokenizer            |       Tokens |     TWR | Chr/Tok |  % Reduction
+----------------------------------------------------------------------
+SGPE                 |    7,240,151 |   1.330 |    4.46 |            -
+OpenAI (o200k_base)  |    7,420,527 |   1.364 |    4.35 |         2.4%
+Llama 4 Scout        |    7,512,843 |   1.381 |    4.30 |         3.6%
+DeepSeek V3          |    7,904,670 |   1.453 |    4.09 |         8.4%
 
-  Test Battery                                           Status         Key Metric
-  ────────────────────────────────────────────────────────────────────────────────
-  Frontier Benchmarking                                  ✓ PASS                   
-  ────────────────────────────────────────────────────────────────────────────────
-  TOTAL                                               P:1  F:0  W:0
-
-  ┌─── Frontier Benchmark Highlight ──────────────────────────────┐
-  │  SGPE TWR:                                 1.438              │
-  │  GPT-4o TWR (o200k_base):                  3.515              │
-  │  SGPE reduction vs GPT-4o:                 59.1%              │
-  │  SGPE reduction vs Llama 4:                60.8%              │
-  └───────────────────────────────────────────────────────────────┘
-
+========================= OVERALL Results =========================
+Tokenizer            |       Tokens |     TWR | Chr/Tok |  % Reduction
+----------------------------------------------------------------------
+SGPE                 |   27,338,091 |   1.241 |    4.47 |            -
+OpenAI (o200k_base)  |   43,174,798 |   1.959 |    2.83 |        36.7%
+Llama 4 Scout        |   45,236,671 |   2.053 |    2.70 |        39.6%
+DeepSeek V3          |   68,739,586 |   3.119 |    1.78 |        60.2%
 
 ================================================================================
 BATTERY 4: ROUND-TRIP CONSISTENCY
 ================================================================================
 
-  Sentences tested:                 536,508
-  Total characters tested:       59,323,178
-  Total tokens generated:        13,256,494
+  Sentences tested:               1,499,950
+  Total words:                   22,190,730
+  Total characters tested:      122,274,117
+  Total tokens generated:        27,503,859
   Mismatches (non-UNK):                   0
-  Mismatches (with UNK loss):        61,350
+  Mismatches (with UNK loss):        19,320
   Crashes:                                0
 
-  Result: PASS — Tested 536,508 sentences (59,323,178 chars). Non-UNK mismatches: 0, UNK-caused losses: 61350, Crashes: 0
-
-  Test Battery                                           Status         Key Metric
-  ────────────────────────────────────────────────────────────────────────────────
-  Round-Trip Consistency (1M sentences)                  ✓ PASS       0 mismatches
-  ────────────────────────────────────────────────────────────────────────────────
-  TOTAL                                               P:1  F:0  W:0
-
+  Result: PASS — Tested 1,499,950 sentences (122,274,117 chars). Non-UNK mismatches: 0, UNK-caused losses: 19320, Crashes: 0
 
 ================================================================================
-BATTERY 5: BOUNDARY & LEADING SPACE EDGE-CASES
+BATTERY 5: BOUNDARY & LEADING SPACE EDGE-CASES 
 ================================================================================
-  Testing whitespace variations...
-  Testing leading spaces before Sinhala...
-  Testing trailing spaces after Sinhala...
-  Testing combined leading/trailing spaces...
-  Testing Sinhala + numbers without spaces...
-  Testing Sinhala + English without spaces...
-  Testing complex mixed boundaries...
-  Testing punctuation boundaries...
-  Testing Unicode edge cases...
-  Testing Leading Space (Ġ) prefix integrity...
-  Testing rapid boundary transitions...
-
-  Result: PASS — Ran 60 edge-case tests. Violations: 0
+  [✓] [B01-Sinhala-leading-space   ] ' සිංහල' -> '[UNK]හල'
+  [✓] [B02-Sinhala-no-leading-space] 'සිංහල' -> '[UNK]හල'
+  [✓] [B03-Sinhala-trailing-punct  ] 'සිංහල.' -> '[UNK]හල.'
+  [✓] [B04-Sinhala-multi-word      ] 'දරුවන් පාසලට' -> 'දරුවන් පාසලට'
+  [✓] [D01-Devanagari-leading-space] ' हिंदी' -> '[UNK]दी'
+  [✓] [D02-Devanagari-no-leading   ] 'नमस्ते' -> 'नमस्ते'
+  [✓] [D03-Devanagari-trailing-danda] 'नमस्ते।' -> 'नमस्ते।'
+  [✓] [D04-Devanagari-multi-word   ] 'भारत देश' -> 'भारत देश'
+  [✓] [D05-Devanagari-anusvara     ] 'संस्कृत' -> 'संस्कृत'
+  [✓] [F01-SinhalaEng              ] 'සිංහලදABC' -> '[UNK]හලදABC'
+  [✓] [F02-DevanagariEng           ] 'हिंदीDEF' -> '[UNK]दीDEF'
+  [✓] [F03-Sinhala-Devanagari      ] 'සිංහල हिंदी' -> '[UNK]හල[UNK]दी'
+  [✓] [G01-Mixed-3-scripts         ] ' සිංහල123ABCहिंदी ' -> '[UNK]හල123ABC[UNK]दी '
 
-  Test Battery                                           Status         Key Metric
-  ────────────────────────────────────────────────────────────────────────────────
-  Boundary & Leading Space Edge-Cases                    ✓ PASS       0 violations
-  ────────────────────────────────────────────────────────────────────────────────
-  TOTAL                                               P:1  F:0  W:0
+  Result: PASS — Violations: 0
 
 ================================================================================
 BATTERY 6: ZERO-BREAKAGE GUARANTEE
@@ -266,8 +229,85 @@ BATTERY 6: ZERO-BREAKAGE GUARANTEE
 
   Result: PASS — Ran 1,703 exhaustive breakage tests. Violations: 0
 
+================================================================================
+BATTERY 6: ZERO-BREAKAGE GUARANTEE (v2 Multi-Script)
+================================================================================
+  Testing Devanagari C + HAL + C pairs (implicit conjuncts)...
+  Testing Devanagari C + vowel_sign...
+  Testing Devanagari C + HAL (terminal virama)...
+  Testing Devanagari C + anusvara / visarga / chandrabindu...
+  Testing Devanagari C + vowel_sign + modifier...
+
+  Result: PASS — Devanagari Violations: 0
+
+================================================================================
+BATTERY 7: DEVANAGARI LINGUISTIC COMPLEXITY
+================================================================================
+
+  Category                      Total   Pass   Fail
+  ----------------------------------------------------
+  anusvara                          1      1      0
+  anusvara_prefix                   5      5      0
+  complex                           2      2      0
+  conjunct                          3      3      0
+  conjunct_anusvara                 4      4      0
+  double_conjunct                   1      1      0
+  double_conjunct_gen             470    470      0
+  extreme_compound                  1      1      0
+  matra                             3      3      0
+  sanskrit                          4      4      0
+  simple                            4      4      0
+  super_compound                    1      1      0
+  very_complex                      1      1      0
+
+  Result: PASS — Tested 500 Devanagari words. Violations: 0
+
+================================================================================
+BATTERY 8: CODE-SWITCHING INTEGRITY
+================================================================================
+  [simple_sinhala_english             ]   5 tokens | ['Hello', ',', ' ශ්\u200dරී', ' ලංකාව', '!']
+  [code_sinhala                       ]   5 tokens | ['const', ' x', ' =', ' ප්\u200dරකාශය', ';']
+  [devanagari_english                 ]   7 tokens | ['मेरा', ' नाम', ' है', ' और', ' I', ' love', ' Python']
+  [code_sinhala_mixed                 ]   9 tokens | ['function', ' foo', '()', ' {', ' return', " '", 'ශ්\u200dරී', "';"]
+  [sinhala_english_mixed              ]   8 tokens | ['ශ', '\u200d', '්', '\u200d', 'රී', ' ලංකාව', ' is', ' beautiful']
+  [python_devanagari_comment          ]   7 tokens | ['print', "('", 'नमस्ते', "')", ' #', ' Say', ' Hello']
+  [sinhala_english_complex            ]   8 tokens | ['ඒ', ' කියන්නේ', ',', ' G', 'PE', ' Token', 'izer', ' English']
+  [python_sinhala_comment             ]  10 tokens | ['for', ' i', ' in', ' range', '(', '10', '):', ' #']
+  [sql_devanagari                     ]   9 tokens | ['SELECT', ' *', ' FROM', ' users', ' WHERE', ' नाम', "='", 'राम']
+  [arrow_fn_sinhala                   ]  22 tokens | ['const', ' create', '_func', ' =', ' (', 'p', '1', ',']
+  [math_sinhala                       ]   6 tokens | ['123', ' +', ' ', '456', ' =', ' ෆ']
+
+  Result: PASS — Tested 13 code-switching cases. Violations: 0, Crashes: 0
+
+================================================================================
+BATTERY 9: META-VOCAB ROUND-TRIP (SGPEMetaEncoder)
+================================================================================
+
+  Sentences:     1,499,950
+  Round-trip failures: 0 (100.00% lossless)
+  Avg tokens/sentence: 18.3
+  UNK rate: 0.08%
+
+  Result: PASS — Tested 1,499,950 sentences. Failures: 0, Crashes: 0, Lossless: 100.00%, UNK rate: 0.08%
+
+
+████████████████████████████████████████████████████████████████████████████████
+█                                                                              █
+█                            SGPE - BATTLE TEST REPORT                         █
+█                                                                              █
+████████████████████████████████████████████████████████████████████████████████
+
   Test Battery                                           Status         Key Metric
   ────────────────────────────────────────────────────────────────────────────────
-  Zero-Breakage Guarantee                              ✓ PASS       0 violations
+  Linguistic Complexity (2K Sanskrit/Pali Words)         ✓ PASS       0 violations
+  Glitched Token Detection (v2)                          ✓ PASS
+  Frontier Benchmarking (Stratified)                     ✓ PASS
+  Round-Trip Consistency (v2)                            ✓ PASS       0 mismatches
+  Boundary Edge-Cases (v2)                               ✓ PASS
+  Zero-Breakage Guarantee (Extended)                     ✓ PASS       0 violations
+  Zero-Breakage Guarantee (v2 Devanagari)                ✓ PASS
+  Devanagari Linguistic Complexity                       ✓ PASS       0 violations
+  Code-Switching Integrity                               ✓ PASS       0 violations
+  Meta-Vocab Round-Trip (SGPEMetaEncoder)                ✓ PASS
   ────────────────────────────────────────────────────────────────────────────────
-  TOTAL                                               P:1  F:0  W:0
+  TOTAL                                              P:10  F:0  W:0

encoder.py

@@ -1,22 +1,30 @@
+"""
+==========================================
+WWHO Encoder  (Unified Meta-Vocabulary)
+==========================================
+"""
+
+from __future__ import annotations
+
 import argparse
 import json
+from typing import Optional
 
-from linguis_trie import LinguisTrie
+from linguis_trie import LinguisTrie, build_sinhala_linguis_trie
 from gpe_trainer import segment_into_words, _is_boundary_token
 
-
 class SGPEEncoder:
 
     def __init__(self, vocab_path: str):
         with open(vocab_path, "r", encoding="utf-8") as f:
             data = json.load(f)
 
-        self.vocab: dict[str, int] = data["vocab"]
-        self.merges: list[tuple[str, str]] = [tuple(m) for m in data["merges"]]
-        self.special_tokens: list[str] = data["special_tokens"]
-        self.tokenizer = LinguisTrie()
-        self.unk_id = self.vocab.get("[UNK]", 1)
-        self.leading_space: bool = data.get("leading_space", False)
+        self.vocab: dict[str, int]             = data["vocab"]
+        self.merges: list[tuple[str, str]]     = [tuple(m) for m in data["merges"]]
+        self.special_tokens: list[str]         = data["special_tokens"]
+        self.tokenizer                         = build_sinhala_linguis_trie()
+        self.unk_id                            = self.vocab.get("[UNK]", 1)
+        self.leading_space: bool               = data.get("leading_space", False)
 
         self._merge_priority: dict[tuple[str, str], int] = {
             (a, b): rank for rank, (a, b) in enumerate(self.merges)
@@ -32,18 +40,15 @@ class SGPEEncoder:
 
         while True:
             best_rank = len(self.merges)
-            best_idx = -1
-
+            best_idx  = -1
             for i in range(len(tokens) - 1):
                 pair = (tokens[i], tokens[i + 1])
                 rank = self._merge_priority.get(pair)
                 if rank is not None and rank < best_rank:
                     best_rank = rank
-                    best_idx = i
-
+                    best_idx  = i
             if best_idx == -1:
                 break
-
             merged = tokens[best_idx] + tokens[best_idx + 1]
             tokens = tokens[:best_idx] + [merged] + tokens[best_idx + 2:]
 
@@ -51,21 +56,17 @@ class SGPEEncoder:
 
     def tokenize(self, text: str) -> list[str]:
         syllables = self.layer1_tokenize(text)
-        words = segment_into_words(syllables)
-
+        words     = segment_into_words(syllables)
         result: list[str] = []
         for word_tokens in words:
             if len(word_tokens) == 1 and _is_boundary_token(word_tokens[0]):
                 result.append(word_tokens[0])
                 continue
-
             cleaned = [t if t in self.vocab else "[UNK]" for t in word_tokens]
             result.extend(self._apply_merges_to_word(cleaned))
-
         return result
 
     def layer1_tokenize(self, text: str) -> list[str]:
-        """Layer 1: Deterministic LinguisTrie pre-tokenization (Syllables)."""
         return self.tokenizer.tokenize(text, leading_space=self.leading_space)
 
     def decode(self, ids: list[int]) -> str:
@@ -73,18 +74,235 @@ class SGPEEncoder:
         return "".join(id_to_token.get(i, "") for i in ids)
 
 
+# ============================================================================
+# MetaVocab — unified ID space
+# ============================================================================
+
+class MetaVocab:
+    def __init__(self, sgpe_vocab: dict[str, int], tiktoken_size: int):
+        self.tiktoken_size: int             = tiktoken_size
+        self._sgpe_raw: dict[str, int]      = sgpe_vocab  
+        self._sgpe_offset: dict[str, int]   = {
+            tok: idx + tiktoken_size for tok, idx in sgpe_vocab.items()
+        }
+        self._sgpe_reverse: dict[int, str]  = {
+            v: k for k, v in self._sgpe_offset.items()
+        }
+
+    @property
+    def total_size(self) -> int:
+        return self.tiktoken_size + len(self._sgpe_raw)
+
+    def encode_sgpe_token(self, token: str, unk_id_raw: int) -> int:
+        return self._sgpe_offset.get(token, unk_id_raw + self.tiktoken_size)
+
+    def decode_id(self, uid: int) -> Optional[str]:
+        if uid < self.tiktoken_size:
+            return None
+        return self._sgpe_reverse.get(uid)
+
+    def is_tiktoken_id(self, uid: int) -> bool:
+        return uid < self.tiktoken_size
+
+    def sgpe_unk_id(self, raw_unk: int) -> int:
+        return raw_unk + self.tiktoken_size
+
+
+# ============================================================================
+# WWHOMetaEncoder 
+# ============================================================================
+
+class WWHOMetaEncoder:
+
+    def __init__(self, vocab_path: str, tiktoken_model: str = "o200k_base"):
+        # Load SGPE vocab
+        with open(vocab_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        sgpe_vocab: dict[str, int]         = data["vocab"]
+        self._merges: list[tuple[str, str]] = [tuple(m) for m in data["merges"]]
+        self._special_tokens: list[str]     = data["special_tokens"]
+        self._leading_space: bool           = data.get("leading_space", False)
+        self._raw_unk_id: int               = sgpe_vocab.get("[UNK]", 1)
+
+        if " " not in sgpe_vocab:
+            next_id = max(sgpe_vocab.values()) + 1
+            sgpe_vocab[" "] = next_id
+
+        try:
+            from router import _INDIC_PUNCT_CHARS
+            for ch in _INDIC_PUNCT_CHARS:
+                if ch not in sgpe_vocab:
+                    next_id = max(sgpe_vocab.values()) + 1
+                    sgpe_vocab[ch] = next_id
+        except ImportError:
+            pass
+
+        self._merge_priority: dict[tuple[str, str], int] = {
+            (a, b): rank for rank, (a, b) in enumerate(self._merges)
+        }
+
+        # tiktoken
+        try:
+            import tiktoken as _tiktoken
+            self._tik = _tiktoken.get_encoding(tiktoken_model)
+        except Exception as e:
+            raise RuntimeError(
+                f"tiktoken ({tiktoken_model!r}) unavailable: {e}. "
+            )
+
+        # Unified vocab
+        self._meta = MetaVocab(sgpe_vocab, self._tik.n_vocab)
+        self._space_id: int = self._meta._sgpe_offset[" "]
+
+        # Router
+        from router import CodeSwitchSegmenter, Script
+        self._segmenter = CodeSwitchSegmenter()
+        self._Script    = Script
+
+        # Indic LinguisTries
+        from linguis_trie import build_sinhala_linguis_trie, build_devanagari_linguis_trie
+        self._sinhala_dfa    = build_sinhala_linguis_trie()
+        self._devanagari_dfa = build_devanagari_linguis_trie()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    @property
+    def vocab_size(self) -> int:
+        return self._meta.total_size
+
+    @property
+    def tiktoken_size(self) -> int:
+        return self._meta.tiktoken_size
+
+    @property
+    def vocab(self) -> dict[str, int]:
+        return self._meta._sgpe_raw
+
+    def encode(self, text: str) -> list[int]:
+        ids: list[int] = []
+        for seg in self._segmenter.segment(text):
+            if seg.script == self._Script.LATIN:
+                ids.extend(self._tik.encode(seg.text))
+            else:
+                dfa = (
+                    self._sinhala_dfa
+                    if seg.script == self._Script.SINHALA
+                    else self._devanagari_dfa
+                )
+                syllables = dfa.tokenize(seg.text, leading_space=seg.has_leading_space)
+                words     = segment_into_words(syllables)
+                for word_toks in words:
+                    if len(word_toks) == 1 and _is_boundary_token(word_toks[0]):
+                        ids.extend(self._tik.encode(word_toks[0]))
+                        continue
+                    merged = self._apply_merges(word_toks)
+                    for tok in merged:
+                        ids.append(self._meta.encode_sgpe_token(tok, self._raw_unk_id))
+        return ids
+
+    def decode(self, ids: list[int]) -> str:
+        tik_buf: list[int] = []
+        result_parts: list[str] = []
+
+        def _flush_tik():
+            if tik_buf:
+                result_parts.append(self._tik.decode(tik_buf))
+                tik_buf.clear()
+
+        for uid in ids:
+            if self._meta.is_tiktoken_id(uid):
+                tik_buf.append(uid)
+            else:
+                _flush_tik()
+                tok = self._meta.decode_id(uid)
+                result_parts.append(tok if tok is not None else "")
+
+        _flush_tik()
+        return "".join(result_parts)
+
+    def tokenize(self, text: str) -> list[str]:
+        tokens: list[str] = []
+        for seg in self._segmenter.segment(text):
+            if seg.script == self._Script.LATIN:
+                ids = self._tik.encode(seg.text)
+                tokens.extend(self._tik.decode([i]) for i in ids)
+            else:
+                dfa = (
+                    self._sinhala_dfa
+                    if seg.script == self._Script.SINHALA
+                    else self._devanagari_dfa
+                )
+                syllables = dfa.tokenize(seg.text, leading_space=seg.has_leading_space)
+                words     = segment_into_words(syllables)
+                for word_toks in words:
+                    if len(word_toks) == 1 and _is_boundary_token(word_toks[0]):
+                        ids = self._tik.encode(word_toks[0])
+                        tokens.extend(self._tik.decode([i]) for i in ids)
+                        continue
+                    tokens.extend(self._apply_merges(word_toks))
+        return tokens
+
+    def _apply_merges(self, tokens: list[str]) -> list[str]:
+        if len(tokens) <= 1:
+            return tokens
+        sgpe = self._meta._sgpe_raw
+        cleaned = [t if t in sgpe else "[UNK]" for t in tokens]
+        while True:
+            best_rank = len(self._merges)
+            best_idx  = -1
+            for i in range(len(cleaned) - 1):
+                pair = (cleaned[i], cleaned[i + 1])
+                rank = self._merge_priority.get(pair)
+                if rank is not None and rank < best_rank:
+                    best_rank = rank
+                    best_idx  = i
+            if best_idx == -1:
+                break
+            merged = cleaned[best_idx] + cleaned[best_idx + 1]
+            cleaned = cleaned[:best_idx] + [merged] + cleaned[best_idx + 2:]
+        return cleaned
+
+
+# ============================================================================
+# CLI
+# ============================================================================
+
 def main():
-    parser = argparse.ArgumentParser(description="SGPE Encoder")
-    parser.add_argument("--vocab", type=str, default="output/vocab.json")
-    parser.add_argument("--text", type=str, required=True)
+    parser = argparse.ArgumentParser(description="WWHO Encoder (Unified Meta-Vocabulary)")
+    parser.add_argument("--vocab",  type=str, default="output/vocab.json",
+                        help="Path to WWHO vocab.json")
+    parser.add_argument("--text",   type=str, required=True,
+                        help="Text to encode (supports mixed Latin + Indic)")
+    parser.add_argument("--mode",   type=str, default="meta",
+                        choices=["sgpe", "meta"],
+                        help="'sgpe' = pure SGPE encoder; 'meta' = unified meta-encoder")
+    parser.add_argument("--tiktoken_model", type=str, default="o200k_base")
     args = parser.parse_args()
 
-    enc = SGPEEncoder(args.vocab)
-    tokens = enc.tokenize(args.text)
-    ids = enc.encode(args.text)
-    print(f"tokens : {tokens}")
-    print(f"ids    : {ids}")
-    print(f"count  : {len(tokens)}")
+    if args.mode == "sgpe":
+        enc    = SGPEEncoder(args.vocab)
+        tokens = enc.tokenize(args.text)
+        ids    = enc.encode(args.text)
+        print(f"[SGPEEncoder]")
+        print(f"  tokens : {tokens}")
+        print(f"  ids    : {ids}")
+        print(f"  count  : {len(tokens)}")
+    else:
+        enc    = WWHOMetaEncoder(args.vocab, tiktoken_model=args.tiktoken_model)
+        tokens = enc.tokenize(args.text)
+        ids    = enc.encode(args.text)
+        decoded = enc.decode(ids)
+        print(f"[WWHOMetaEncoder]")
+        print(f"  vocab_size    : {enc.vocab_size:,}  "
+              f"(tiktoken={enc.tiktoken_size:,} + SGPE={enc.vocab_size - enc.tiktoken_size:,})")
+        print(f"  tokens : {tokens}")
+        print(f"  ids    : {ids}")
+        print(f"  count  : {len(tokens)}")
+        print(f"  decoded: {decoded!r}")
+        print(f"  lossless: {decoded == args.text}")
 
 
 if __name__ == "__main__":

linguis_trie.py

@@ -1,119 +1,146 @@
 """
 ==========================================
-SGPE Layer 1 — LinguisTrie Pre-tokenizer
+Table-Driven DFA Tokenizer
 ==========================================
 """
 
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+# ---------------------------------------------------------------------------
+# Schema loading and validation
+# ---------------------------------------------------------------------------
+
+class SchemaError(ValueError):
+    """Raised when a language schema JSON is malformed or incomplete."""
+
+@dataclass
+class LanguageSchema:
+    language: str
+    grammar_notation: str
+    char_classes: dict[str, set[int]]         # class-label → set of codepoints
+    transitions: dict[str, dict[str, Optional[str]]]  # state → (class → next_state | None)
+    start_state: str
+    accept_states: set[str]
+    emit_states: set[str]
+
+    def get_regex(self) -> str:
+        parts = []
+        for cps in self.char_classes.values():
+            for cp in cps:
+                parts.append(chr(cp))
+        
+        if not parts:
+            return ""
+            
+        safe_parts = []
+        for p in parts:
+            if p in ('-', ']', '\\', '^'):
+                safe_parts.append('\\' + p)
+            else:
+                safe_parts.append(p)
+                
+        char_set = "".join(set(safe_parts))
+        return f"[{char_set}]+"
+
 
+class SchemaLoader:
+    def load(self, path: str) -> LanguageSchema:
+        with open(path, "r", encoding="utf-8") as fh:
+            raw = json.load(fh)
 
-HAL = '\u0DCA'   # ්  virama / al-lakuna
-ZWJ = '\u200D'   # zero-width joiner
-
-# --- Independent vowels (svara) ---
-VOWELS: set[str] = {
-    '\u0D85',  # අ
-    '\u0D86',  # ආ
-    '\u0D87',  # ඇ
-    '\u0D88',  # ඈ
-    '\u0D89',  # ඉ
-    '\u0D8A',  # ඊ
-    '\u0D8B',  # උ
-    '\u0D8C',  # ඌ
-    '\u0D8D',  # ඍ
-    '\u0D8E',  # ඎ
-    '\u0D8F',  # ඏ
-    '\u0D90',  # ඐ
-    '\u0D91',  # එ
-    '\u0D92',  # ඒ
-    '\u0D93',  # ඓ
-    '\u0D94',  # ඔ
-    '\u0D95',  # ඕ
-    '\u0D96',  # ඖ
-}
-
-# --- Consonants (vyanjana) ---
-CONSONANTS: set[str] = {chr(c) for c in range(0x0D9A, 0x0DC7)}
-
-# --- Dependent vowel signs (pili) ---
-VOWEL_SIGNS: set[str] = {
-    '\u0DCF',  # ා
-    '\u0DD0',  # ැ
-    '\u0DD1',  # ෑ
-    '\u0DD2',  # ි
-    '\u0DD3',  # ී
-    '\u0DD4',  # ු
-    '\u0DD5',  # ෕ (rare/archaic)
-    '\u0DD6',  # ූ
-    '\u0DD7',  # ෗ (rare/archaic)
-    '\u0DD8',  # ෘ
-    '\u0DD9',  # ෙ
-    '\u0DDA',  # ේ
-    '\u0DDB',  # ෛ
-    '\u0DDC',  # ො
-    '\u0DDD',  # ෝ
-    '\u0DDE',  # ෞ
-    '\u0DDF',  # ෟ
-    '\u0DF2',  # ෲ
-    '\u0DF3',  # ෳ
-}
-
-# --- Post-consonant modifiers (anusvara, visarga) ---
-POST_MODIFIERS: set[str] = {
-    '\u0D82',  # ං  anusvara
-    '\u0D83',  # ඃ  visarga
-}
-
-
-
-def _is_consonant(ch: str) -> bool:
-    return ch in CONSONANTS
-
-def _is_vowel(ch: str) -> bool:
-    return ch in VOWELS
-
-def _is_vowel_sign(ch: str) -> bool:
-    return ch in VOWEL_SIGNS
-
-def _is_post_modifier(ch: str) -> bool:
-    return ch in POST_MODIFIERS
-
-def _is_hal(ch: str) -> bool:
-    return ch == HAL
-
-def _is_zwj(ch: str) -> bool:
-    return ch == ZWJ
-
-def _is_sinhala(ch: str) -> bool:
-    """Any character in the Sinhala Unicode block or ZWJ."""
-    cp = ord(ch)
-    return (0x0D80 <= cp <= 0x0DFF) or cp == 0x200D
+        language = raw.get("language", "unknown")
+        grammar  = raw.get("grammar_notation", "")
 
+        if "char_classes" not in raw:
+            raise SchemaError(f"[{path}] Missing 'char_classes' key.")
+        if "dfa" not in raw:
+            raise SchemaError(f"[{path}] Missing 'dfa' key.")
 
+        char_classes: dict[str, set[int]] = {}
+        for label, definition in raw["char_classes"].items():
+            if label.startswith("_"):
+                continue
+            cps: set[int] = set()
+            for rng in definition.get("ranges", []):
+                lo, hi = int(rng[0], 16), int(rng[1], 16)
+                cps.update(range(lo, hi + 1))
+            for cp_hex in definition.get("codepoints", []):
+                cps.add(int(cp_hex, 16))
+            char_classes[label] = cps
+
+        dfa_raw = raw["dfa"]
+        start_state   = dfa_raw.get("start", "START")
+        accept_states = set(dfa_raw.get("accept_states", []))
+        emit_states   = set(dfa_raw.get("emit_states", []))
+        transitions   = dfa_raw.get("transitions", {})
+
+        return LanguageSchema(
+            language=language,
+            grammar_notation=grammar,
+            char_classes=char_classes,
+            transitions=transitions,
+            start_state=start_state,
+            accept_states=accept_states,
+            emit_states=emit_states,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Codepoint classifier
+# ---------------------------------------------------------------------------
+
+class CharClassifier:
+    def __init__(self, schema: LanguageSchema):
+        self._table: dict[int, str] = {}
+        for label, cps in schema.char_classes.items():
+            for cp in cps:
+                if cp in self._table:
+                    continue
+                self._table[cp] = label
+
+    def classify(self, ch: str) -> str:
+        return self._table.get(ord(ch), "O")
+
+
+# ---------------------------------------------------------------------------
+# DFA Tokenizer
+# ---------------------------------------------------------------------------
 
 class LinguisTrie:
+    def __init__(self, schema: LanguageSchema):
+        self._schema      = schema
+        self._classifier  = CharClassifier(schema)
+        self._transitions = schema.transitions
+        self._start       = schema.start_state
+        self._accept      = schema.accept_states
+        self._emit        = schema.emit_states
+
     def tokenize(self, text: str, leading_space: bool = False) -> list[str]:
-        """
-        Tokenize Sinhala text into atomic syllable tokens.
-        Example: "මම යනවා" → [" මම", " ය", "න", "වා"]
-        """
         tokens: list[str] = []
-        n = len(text)
-        pos = 0
-        pending_space = ""
+        n     = len(text)
+        pos   = 0
+        
+        pending_space = " " if leading_space and text and text[0] not in (" ", "\t", "\n", "\r") else ""
 
         while pos < n:
             ch = text[pos]
 
-            # ─── Whitespace handling (leading-space mode) ─────────
-            if leading_space and ch in (' ', '\t', '\n', '\r'):
+            # ─── Whitespace handling (leading-space mode) ────────────
+            if leading_space and ch in (" ", "\t", "\n", "\r"):
                 ws_buffer = ""
-                while pos < n and text[pos] in (' ', '\t', '\n', '\r'):
+                while pos < n and text[pos] in (" ", "\t", "\n", "\r"):
                     ws_buffer += text[pos]
                     pos += 1
-                
-                if ws_buffer.endswith(' '):
+
+                if ws_buffer.endswith(" "):
                     for ws_char in ws_buffer[:-1]:
-                         tokens.append(ws_char)
+                        tokens.append(ws_char)
                     pending_space = " "
                 else:
                     for ws_char in ws_buffer:
@@ -121,92 +148,119 @@ class LinguisTrie:
                     pending_space = ""
                 continue
 
-            # ─── Consonant-initiated syllable ─────────────────────
-            if _is_consonant(ch):
-                start = pos
-                pos += 1
+            # ─── DFA syllable recognition ────────────────────
+            cls       = self._classifier.classify(ch)
+            init_next = self._transitions.get(self._start, {}).get(cls)
 
-                # Absorb consonant cluster: (HAL [ZWJ] Consonant)*
-                #   Handles: C්C (implicit), C්‍C (ZWJ), and stacks
-                while pos < n and _is_hal(text[pos]):
-                    if pos + 1 < n and _is_zwj(text[pos + 1]):
-                        # HAL + ZWJ: must be followed by consonant
-                        if pos + 2 < n and _is_consonant(text[pos + 2]):
-                            pos += 3  
-                            continue
-                        else:
-                            # Stray HAL+ZWJ at end — absorb HAL+ZWJ
-                            pos += 2
-                            break
-
-                    elif pos + 1 < n and _is_consonant(text[pos + 1]):
-                        # HAL + C (implicit conjunct, no ZWJ)
-                        pos += 2  
-                        continue
-
-                    else:
-                        break
-
-                # ── Post-cluster modifiers ──
-
-                if pos < n and _is_vowel_sign(text[pos]):
-                    pos += 1   # pili
-                elif pos < n and _is_hal(text[pos]):
-                    pos += 1   # virama
-
-                if pos < n and _is_post_modifier(text[pos]):
-                    pos += 1   # anusvara/visarga
-
-                tokens.append(pending_space + text[start:pos])
-                pending_space = ""
-                continue
-
-            # ─── Independent vowel ────────────────────────────────
-            if _is_vowel(ch):
-                start = pos
+            if init_next is None:
+                if pending_space:
+                    tokens.append(pending_space + ch)
+                    pending_space = ""
+                else:
+                    tokens.append(ch)
                 pos += 1
-
-                # Vowel + post-modifier (e.g. අං)
-                if pos < n and _is_post_modifier(text[pos]):
-                    pos += 1
-
-                tokens.append(pending_space + text[start:pos])
-                pending_space = ""
                 continue
 
-            # ─── Orphan post-modifier ──
-            if _is_post_modifier(ch) or _is_hal(ch) or _is_vowel_sign(ch):
+            if init_next in self._emit:
                 tokens.append(pending_space + ch)
                 pending_space = ""
                 pos += 1
                 continue
 
-            # ─── Non-Sinhala passthrough (punctuation, digits, etc.) ──
-            if pending_space:
-                tokens.append(pending_space + ch)
-                pending_space = ""
+            span_start = pos
+            state      = init_next
+            pos       += 1
+            last_accept_pos = pos if state in self._accept else -1
+
+            while pos < n:
+                ch2  = text[pos]
+                cls2 = self._classifier.classify(ch2)
+                next_state = self._transitions.get(state, {}).get(cls2)
+
+                if next_state is None:
+                    break
+
+                state = next_state
+                pos  += 1
+
+                if state in self._accept:
+                    last_accept_pos = pos
+                elif state in self._emit:
+                    last_accept_pos = pos
+                    break
+
+            if last_accept_pos > span_start:
+                emit_end = last_accept_pos
             else:
-                tokens.append(ch)
-            pos += 1
+                emit_end = pos
+
+            tokens.append(pending_space + text[span_start:emit_end])
+            pending_space = ""
+            pos = emit_end
 
         if pending_space:
             tokens.append(pending_space)
 
         return tokens
 
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @property
+    def language(self) -> str:
+        return self._schema.language
+
+    @property
+    def regex(self) -> str:
+        return self._schema.get_regex()
+
+    @property
+    def grammar(self) -> str:
+        return self._schema.grammar_notation
+
+
+# ---------------------------------------------------------------------------
+# Factory
+# ---------------------------------------------------------------------------
+
+_SCHEMA_DIR = os.path.join(os.path.dirname(__file__), "schemas")
 
+_schema_loader = SchemaLoader()
+_dfa_cache: dict[str, LinguisTrie] = {}
 
-def build_linguistrie() -> LinguisTrie:
-    """Build and return the LinguisTrie."""
-    return LinguisTrie()
 
+def build_linguis_trie(schema_path: str) -> LinguisTrie:
+    if schema_path not in _dfa_cache:
+        schema = _schema_loader.load(schema_path)
+        _dfa_cache[schema_path] = LinguisTrie(schema)
+    return _dfa_cache[schema_path]
 
 
-if __name__ == '__main__':
-    trie = build_linguistrie()
+def build_sinhala_linguis_trie() -> LinguisTrie:
+    return build_linguis_trie(os.path.join(_SCHEMA_DIR, "sinhala.json"))
 
-    test_sentences = [
-        # Core tests from the plan
+
+def build_devanagari_linguis_trie() -> LinguisTrie:
+    return build_linguis_trie(os.path.join(_SCHEMA_DIR, "devanagari.json"))
+
+
+# ---------------------------------------------------------------------------
+# Self-test
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import sys
+
+    print("=" * 65)
+    print("DFA Tokenizer — self-test")
+    print("=" * 65)
+
+    # --- Sinhala ---
+    sinhala_dfa = build_sinhala_linguis_trie()
+    print(f"\n[Sinhala DFA]  grammar: {sinhala_dfa.grammar}\n")
+
+    sinhala_tests = [
         "ශ්‍රී ලංකා ද්වීපයේ ස්වෛරීභාවය සහ ත්‍රිවිධ හමුදාව.",
         "භාෂාවේ ප්‍රෞඪත්වය විදහාපායි",
         "ආචාර්යවරයාගේ වෛද්‍ය විද්‍යා පර්යේෂණය සාර්ථකයි.",
@@ -214,13 +268,42 @@ if __name__ == '__main__':
         "මම ක්‍ෂණිකව ගඟට පැන්නා",
         "සඤ්ඤක ක්ෂමතාවය ක්‍රමය සහ ඥානය",
         "ද්වී ත්වේ ලං කඃ",
-        "න්ද්‍රී ක්ෂි ඤ්ඤ",
         "2026 වසරේ AI තාක්ෂණය 60% දියුණුයි!",
     ]
 
-    for text in test_sentences:
-        tokens = trie.tokenize(text)
-        print(f"Input:  {text}")
-        print(f"Tokens: {tokens}")
-        print(f"Count:  {len(tokens)}")
-        print("-" * 60)
+    for text in sinhala_tests:
+        toks = sinhala_dfa.tokenize(text, leading_space=True)
+        print(f"  Input : {text}")
+        print(f"  Syllables: {toks}")
+        print(f"  Count : {len(toks)}")
+        print("-" * 65)
+
+    # --- Devanagari ---
+    deva_dfa = build_devanagari_linguis_trie()
+    print(f"\n[Devanagari DFA]  grammar: {deva_dfa.grammar}\n")
+
+    deva_tests = [
+        "नमस्ते",
+        "भारत",
+        "हिन्दी",
+        "संस्कृत",
+        "क़िला",
+        "ज़िंदगी",
+        "प्रेम",
+        "द्वारा",
+        "श्रीमान्",
+        "हिन्दुस्तान",
+        "नमस्कार दुनिया",
+        "मैं ठीक हूँ",
+        "विद्यालय में पढ़ाई होती है।",
+    ]
+
+    for text in deva_tests:
+        toks = deva_dfa.tokenize(text, leading_space=True)
+        print(f"  Input : {text}")
+        print(f"  Syllables: {toks}")
+        print(f"  Count : {len(toks)}")
+        print("-" * 65)
+
+    print("\nAll self-tests complete.")
+    sys.exit(0)

meta_config.json

@@ -0,0 +1,8 @@
+{
+  "tiktoken_model": "o200k_base",
+  "tiktoken_vocab_size": 200019,
+  "sgpe_vocab_size": 128000,
+  "sgpe_id_offset": 200019,
+  "script_mode": "mixed",
+  "sgpe_vocab_path": "vocab.json"
+}

router.py

@@ -0,0 +1,247 @@
+"""
+==========================================
+Code-Switching Router
+==========================================
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Optional
+
+import tiktoken
+
+from linguis_trie import build_sinhala_linguis_trie, build_devanagari_linguis_trie, LinguisTrie
+
+
+# ---------------------------------------------------------------------------
+# Script-block detection
+# ---------------------------------------------------------------------------
+
+class Script(Enum):
+    LATIN  = auto()   # ASCII, Latin, digits, punctuation, code, emoji, etc.
+    SINHALA     = auto()
+    DEVANAGARI  = auto()
+
+_sinhala_dfa    = build_sinhala_linguis_trie()
+_devanagari_dfa = build_devanagari_linguis_trie()
+
+_INDIC_PUNCT_CHARS = "\u0964\u0965"
+
+def _get_char_script(ch: str) -> Optional[Script]:
+    if '\u0D80' <= ch <= '\u0DFF':
+        return Script.SINHALA
+    if '\u0900' <= ch <= '\u097F':
+        return Script.DEVANAGARI
+    if ch in _INDIC_PUNCT_CHARS:
+        return Script.SINHALA  # Dandas handled identically by both schemas
+    return None
+
+def _is_indic_joiner(ch: str) -> bool:
+    # True if ZWJ or ZWNJ
+    return ch in ('\u200C', '\u200D')
+
+
+# ---------------------------------------------------------------------------
+# Segment dataclass
+# ---------------------------------------------------------------------------
+
+@dataclass
+class TextSegment:
+    text: str
+    script: Script
+    has_leading_space: bool = False   # True if a boundary space was absorbed
+
+
+# ---------------------------------------------------------------------------
+# Segmenter
+# ---------------------------------------------------------------------------
+
+class CodeSwitchSegmenter:
+    def segment(self, text: str) -> list[TextSegment]:
+        if not text:
+            return []
+
+        segments: list[TextSegment] = []
+        n = len(text)
+        pos = 0
+
+        while pos < n:
+            ch = text[pos]
+            ch_script = _get_char_script(ch)
+
+            is_indic_start = (ch_script is not None)
+
+            if not is_indic_start:
+                # ─── 1. Accumulate Latin block ───
+                start = pos
+                while pos < n:
+                    ch2 = text[pos]
+                    if _get_char_script(ch2) is not None:
+                        break  # Found distinct Indic start
+                    pos += 1
+                
+                latin_chunk = text[start:pos]
+                
+                has_ls = False
+                if pos < n and latin_chunk.endswith(" "):
+                    latin_chunk = latin_chunk[:-1]
+                    has_ls = True
+                
+                if latin_chunk:
+                    segments.append(TextSegment(text=latin_chunk, script=Script.LATIN))
+
+                if has_ls and pos < n:
+                    indic_start = pos
+                    current_script = _get_char_script(text[pos]) or Script.SINHALA
+                    
+                    while pos < n:
+                        c = text[pos]
+                        c_script = _get_char_script(c)
+                        if _is_indic_joiner(c):
+                            pos += 1
+                        elif c_script is not None:
+                            if c_script != current_script and c not in _INDIC_PUNCT_CHARS:
+                                break
+                            pos += 1
+                        else:
+                            break
+                            
+                    segments.append(TextSegment(
+                        text=text[indic_start:pos],
+                        script=current_script,
+                        has_leading_space=True
+                    ))
+            else:
+                # ─── 2. Accumulate Indic block (no prior Latin with space) ───
+                indic_start = pos
+                current_script = ch_script
+                
+                while pos < n:
+                    c = text[pos]
+                    c_script = _get_char_script(c)
+                    if _is_indic_joiner(c):
+                        pos += 1
+                    elif c_script is not None:
+                        if c_script != current_script and c not in _INDIC_PUNCT_CHARS:
+                            break
+                        pos += 1
+                    else:
+                        break
+                        
+                segments.append(TextSegment(
+                    text=text[indic_start:pos],
+                    script=current_script,
+                    has_leading_space=False
+                ))
+
+        return segments
+
+
+
+# ---------------------------------------------------------------------------
+# Router
+# ---------------------------------------------------------------------------
+
+class CodeSwitchRouter:
+    def __init__(
+        self,
+        tiktoken_model: str = "o200k_base",
+        sinhala_schema: Optional[str] = None,
+        devanagari_schema: Optional[str] = None,
+    ):
+        # Indic DFAs
+        self._sinhala_dfa:    LinguisTrie = build_sinhala_linguis_trie()
+        self._devanagari_dfa: LinguisTrie = build_devanagari_linguis_trie()
+
+        self._enc = tiktoken.get_encoding(tiktoken_model)
+
+        self._segmenter = CodeSwitchSegmenter()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def tokenize_to_strings(self, text: str) -> list[str]:
+        result: list[str] = []
+        for seg in self._segmenter.segment(text):
+            result.extend(self._route_segment_strings(seg))
+        return result
+
+    def tokenize_to_ids(self, text: str) -> list[int]:
+        raise NotImplementedError(
+            "Use WWHOMetaEncoder.encode() for unified IDs. "
+            "tokenize_to_ids() on the raw router is intentionally not implemented "
+            "to prevent accidental ID space collision."
+        )
+
+        return self._enc.encode(text)
+
+    def tiktoken_decode(self, ids: list[int]) -> str:
+        return self._enc.decode(ids)
+
+    def tiktoken_vocab_size(self) -> int:
+        return self._enc.n_vocab
+
+    # ------------------------------------------------------------------
+    # Internal routing
+    # ------------------------------------------------------------------
+
+    def _route_segment_strings(self, seg: TextSegment) -> list[str]:
+        if seg.script == Script.LATIN:
+            ids = self._enc.encode(seg.text)
+            return [self._enc.decode([i]) for i in ids]
+
+        # Indic — route to appropriate DFA
+        dfa = (
+            self._sinhala_dfa
+            if seg.script == Script.SINHALA
+            else self._devanagari_dfa
+        )
+        return dfa.tokenize(seg.text, leading_space=seg.has_leading_space)
+
+
+# ---------------------------------------------------------------------------
+# Self-test
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    router = CodeSwitchRouter()
+
+    test_cases = [
+        # Pure Sinhala
+        "ශ්‍රී ලංකාව",
+        # Pure English
+        "Hello, world!",
+        # Mixed — English then Sinhala
+        "The capital is කොළඹ.",
+        # Mixed — Sinhala then English
+        "ලංකාව is beautiful.",
+        # Mixed — Devanagari
+        "Hello नमस्ते world",
+        # Code-switching with numbers
+        "2026 AI සහ machine learning",
+        # Boundary space edge-case
+        "GPT-4 ශ්‍රී ලංකා",
+        # Dense Sinhala
+        "ආචාර්යවරයාගේ වෛද්‍ය විද්‍යා පර්යේෂණය සාර්ථකයි.",
+        # Dense Devanagari
+        "विद्यालय में पढ़ाई होती है।",
+        # Multi-script sentence
+        "AI (Artificial Intelligence) සහ देवनागरी text.",
+    ]
+
+    print("=" * 70)
+    print("CodeSwitchRouter — self-test")
+    print("=" * 70)
+
+    seg = CodeSwitchSegmenter()
+    for text in test_cases:
+        tokens = router.tokenize_to_strings(text)
+        blocks = seg.segment(text)
+        print(f"\n  Input  : {text!r}")
+        print(f"  Blocks : {[(b.text, b.script.name, b.has_leading_space) for b in blocks]}")
+        print(f"  Tokens : {tokens}")
+        print(f"  Count  : {len(tokens)}")