Publish SamAI Tokenizer

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+.venv/
+results/generated/

CITATION.cff

@@ -0,0 +1,10 @@
+cff-version: 1.2.0
+message: "If you use SunAI-64K, please cite this repository."
+title: "SunAI-64K: An Arabic-Focused Byte-Level BPE Tokenizer"
+type: software
+authors:
+  - family-names: Aljohani
+    given-names: Yasser
+version: 1.0.0
+date-released: 2026-09-20
+license: Apache-2.0

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,7 @@
+SunAI-64K
+Copyright 2026 Yasser Aljohani
+
+This product includes aggregate benchmark results produced from third-party
+datasets and tokenizers. Those projects remain governed by their respective
+licenses and terms. No benchmark source documents or training-corpus text are
+redistributed in this repository.

PUBLISHING.md

@@ -0,0 +1,66 @@
+# Publish to Hugging Face
+
+The repository is prepared as a Hugging Face **model repository** containing a
+tokenizer, not model weights.
+
+## 1. Review the public metadata
+
+Before publishing, replace every occurrence of `YOUR_USERNAME` in `README.md`,
+`README_AR.md`, and `example.py` with your Hugging Face username or organization.
+Confirm the project name and author details in `CITATION.cff`.
+
+## 2. Authenticate
+
+```bash
+python -m pip install -U huggingface_hub
+hf auth login
+hf auth whoami
+```
+
+Use a Hugging Face account or token with write permission. Never place the
+token inside a source file or commit it to the repository.
+
+## 3. Verify locally
+
+```bash
+python -m pip install -r requirements.txt
+python verify_tokenizer.py
+sha256sum -c SHA256SUMS
+```
+
+## 4. Upload
+
+```bash
+python upload_to_hub.py YOUR_USERNAME/SunAI-64K
+```
+
+The script creates a public model repository if it does not already exist and
+uploads the folder in one commit. To test privately first, add `--private`.
+
+You can also use the Hugging Face CLI after creating the repository:
+
+```bash
+hf upload YOUR_USERNAME/SunAI-64K . .
+```
+
+## 5. Test the published repository
+
+```python
+from transformers import PreTrainedTokenizerFast
+
+repo_id = "YOUR_USERNAME/SunAI-64K"
+tokenizer = PreTrainedTokenizerFast.from_pretrained(repo_id)
+
+text = "المملكة العربية السعودية"
+ids = tokenizer.encode(text, add_special_tokens=False)
+
+assert tokenizer.decode(ids) == text
+assert len(tokenizer) == 65_536
+print(ids)
+```
+
+## Recommended first release
+
+After verifying the public page, create a `v1.0.0` tag so downstream training
+runs can pin the tokenizer revision instead of depending on a moving `main`
+branch.

README.md

@@ -0,0 +1,232 @@
+---
+language:
+  - ar
+  - en
+license: apache-2.0
+library_name: tokenizers
+datasets:
+  - AdaMLLab/AraMix
+  - HuggingFaceFW/fineweb-2
+  - HuggingFaceFW/fineweb
+tags:
+  - tokenizer
+  - arabic
+  - arabic-nlp
+  - bpe
+  - byte-level
+  - saudi-arabia
+---
+
+# SamAI-Tokenizer
+
+SamAI-Tokenizer is an Arabic-focused, byte-level BPE tokenizer intended for training
+new Arabic-heavy language models. It has a vocabulary of 65,536 tokens, uses
+NFC normalization, preserves the original text during decoding, and includes
+reserved tokens for future model development.
+
+This repository contains the complete tokenizer, validation reports, the
+four-dataset benchmark implementation, and aggregate benchmark results. It
+does **not** contain model weights or redistribute training/benchmark text.
+
+## Key specifications
+
+| Property | Value |
+| --- | --- |
+| Algorithm | Byte-level BPE |
+| Vocabulary size | 65,536 |
+| Normalization | Unicode NFC |
+| Pre-tokenizer | ByteLevel |
+| Unknown-token fallback | Byte-level coverage |
+| Named special tokens | `<pad>`, `<bos>`, `<eos>`, `<unk>`, `<user>`, `<assistant>`, `<system>` |
+| Reserved token IDs | 7–255 |
+| Tokenizer SHA-256 | `7d9496a251b44688557a79fab59b5cf7a12f11a9559540b9a1e223657f35f613` |
+
+## Installation
+
+```bash
+pip install -U tokenizers transformers huggingface_hub
+```
+
+## Usage
+
+Replace `SamAI/SamAI-Tokenizer` with the published repository ID.
+
+### Transformers
+
+```python
+from transformers import PreTrainedTokenizerFast
+
+tokenizer = PreTrainedTokenizerFast.from_pretrained(
+    "SamAI/SamAI-Tokenizer"
+)
+
+text = "المملكة العربية السعودية"
+ids = tokenizer.encode(text, add_special_tokens=False)
+
+print(ids)
+print(tokenizer.convert_ids_to_tokens(ids))
+print(tokenizer.decode(ids))
+```
+
+### Tokenizers
+
+```python
+from huggingface_hub import hf_hub_download
+from tokenizers import Tokenizer
+
+path = hf_hub_download(
+    repo_id="SamAI/SamAI-Tokenizer",
+    filename="tokenizer.json",
+)
+tokenizer = Tokenizer.from_file(path)
+
+encoding = tokenizer.encode(
+    "السلام عليكم ورحمة الله وبركاته",
+    add_special_tokens=False,
+)
+
+print(encoding.ids)
+print(encoding.tokens)
+print(tokenizer.decode(encoding.ids))
+```
+
+## Training mixture
+
+The tokenizer was trained from sampled text covering Arabic and English. The
+training text itself is not included.
+
+| Source bucket | Characters used |
+| --- | ---: |
+| Saudi schoolbooks | 29,856,611 |
+| AraMix Arabic | 200,000,000 |
+| FineWeb2 Arabic | 200,000,000 |
+| FineWeb English | 200,000,000 |
+| **Total** | **629,856,611** |
+
+The released metadata records the training-input SHA-256 as
+`bf02ef3cfa689f435aa5402ace1cb09221f9bec22b1f19dd6e73eaf1bddbd115`.
+This identifies the exact processed input used by the original training run;
+it is not a hash of the upstream datasets.
+
+## Validation
+
+The supplied validation run reported zero round-trip failures and zero unknown
+tokens across Saudi schoolbooks, AraMix Arabic, FineWeb2 Arabic, and FineWeb
+English samples. See [`reports/validation.json`](reports/validation.json) and
+[`reports/stress.json`](reports/stress.json).
+
+## Four-dataset Arabic benchmark
+
+SamAI-Tokenizer was compared on four complementary Arabic datasets:
+
+| Benchmark | Register | Documents | Words |
+| --- | --- | ---: | ---: |
+| ArabicMMLU | Academic questions and reading passages | 14,435 | 410,566 |
+| MLQA Arabic | Questions and long answer-bearing passages | 10,933 | 638,708 |
+| MASSIVE Arabic | Short assistant commands | 2,907 | 15,356 |
+| Arabic Dialects Dataset | Regional dialect text | 9,807 | 337,653 |
+| **Combined** |  | **38,082** | **1,402,283** |
+
+The primary ranking is an equal-weight macro average: each benchmark receives
+25% weight regardless of corpus size. Lower tokens per word is better.
+
+| Rank | Tokenizer | Mean tokens/word | Tokens versus SunAI |
+| ---: | --- | ---: | ---: |
+| 1 | ALLaM-7B | 1.5021 | -4.86% |
+| **2** | **SamAI-Tokenizer** | **1.6207** | **baseline** |
+| 3 | Falcon-H1-7B | 1.8880 | +19.30% |
+| 4 | Mistral-Small-3.1 | 1.8929 | +20.71% |
+| 5 | Qwen3.8-Flash-Next | 1.9039 | +21.94% |
+| 6 | Qwen3.5-4B | 1.9039 | +21.94% |
+| 7 | tiktoken-o200k | 1.9919 | +26.37% |
+| 8 | GPT-OSS-20B | 1.9919 | +26.37% |
+| 9 | Fanar-1 | 2.1570 | +36.81% |
+| 10 | DeepSeek-V3 | 2.2774 | +46.25% |
+| 11 | Qwen2.5-7B | 2.3740 | +50.25% |
+| 12 | Qwen3-8B | 2.3740 | +50.25% |
+| 13 | GLM-5.3-Flash | 2.5225 | +60.21% |
+| 14 | Kimi-K2.6 | 2.8693 | +84.13% |
+| 15 | tiktoken-cl100k | 4.0800 | +164.63% |
+
+### Results by benchmark
+
+| Benchmark | SunAI tokens/word | Best tokenizer | Best tokens/word | SunAI position |
+| --- | ---: | --- | ---: | ---: |
+| Arabic Dialects | 1.3043 | ALLaM-7B | 1.2812 | 2 |
+| ArabicMMLU | 2.4838 | ALLaM-7B | 1.9943 | 5 |
+| MASSIVE Arabic | 1.2549 | ALLaM-7B | 1.1939 | 2 |
+| MLQA Arabic | **1.4397** | **SamAI-Tokenizer** | **1.4397** | **1** |
+
+Complete aggregate results are under [`results/`](results/). The row-level
+benchmark output is intentionally excluded because it reproduces third-party
+dataset text; the script can regenerate it for users who accept the applicable
+dataset terms.
+
+## Benchmark methodology
+
+- All available examples were used, followed by exact-text deduplication within
+  each benchmark.
+- Token counts exclude chat templates and added special tokens.
+- `tokens_per_word` is total tokens divided by whitespace-separated words.
+- The macro ranking averages each benchmark's ratio to SunAI, giving every
+  benchmark equal weight.
+- The pooled report instead sums all tokens and words before calculating the
+  ratio; MLQA and ArabicMMLU therefore receive more weight in that view.
+- Qwen2.5 and Qwen3 produced identical results. Qwen3.5 and Qwen3.8 also
+  produced identical results. GPT-OSS and `o200k_base` produced identical
+  results. These should be interpreted as tokenizer-family duplicates, not
+  independent confirmations.
+- Jais-13B, Gemma-3-4B, Llama-4-Scout, and Aya-Expanse were gated during the
+  published run. AceGPT-v2 could not be loaded by the tested library version.
+  They are not included in the ranking.
+
+To reproduce the benchmark:
+
+```bash
+pip install -r benchmark/requirements.txt
+python benchmark/benchmark.py
+```
+
+Some competitor tokenizers require executing repository-provided Python code.
+That is disabled by default. Review and pin the relevant repositories before
+enabling it:
+
+```bash
+SUNAI_TRUST_REMOTE_CODE=1 python benchmark/benchmark.py
+```
+
+## Limitations
+
+- Tokenization efficiency does not measure language-model quality, factuality,
+  safety, reasoning, or downstream accuracy.
+- Fully vocalized Arabic is the clearest weakness. One heavily diacritized
+  ArabicMMLU subject accounts for most of the gap to ALLaM. Excluding `Arabic
+  Language (General)`, SunAI records approximately 1.630 tokens/word versus
+  ALLaM's 1.709 on the remaining ArabicMMLU subjects.
+- ArabicMMLU contains reading passages repeated across different questions.
+  Exact full-document deduplication does not remove every repeated passage.
+- Whitespace-separated words are a transparent efficiency denominator, but
+  they are not a linguistic segmentation of Arabic morphology.
+- The tokenizer was designed for training new models. Replacing the tokenizer
+  of an already pretrained model requires resizing and retraining its embedding
+  and output layers; it is not a drop-in vocabulary replacement.
+
+## License and data terms
+
+The tokenizer files and repository code are released under Apache-2.0. Source
+datasets and benchmark datasets remain subject to their own licenses and terms.
+No source-corpus or benchmark documents are distributed here. Users are
+responsible for confirming that their intended use complies with the relevant
+upstream terms.
+
+## Citation
+
+```bibtex
+@software{aljohani2026sunai64k,
+  author  = {Yasser Aljohani},
+  title   = {SamAI-Tokenizer: An Arabic-Focused Byte-Level BPE Tokenizer},
+  year    = {2026},
+  version = {1.0.0},
+  url     = {https://huggingface.co/SamAI/SamAI-Tokenizer}
+}
+```

README_AR.md

@@ -0,0 +1,39 @@
+# SamAI-Tokenizer
+
+`SamAI-Tokenizer` هو مقطّع نصوص عربي مبني بخوارزمية BPE على مستوى البايت، ومصمم
+لتدريب نماذج لغوية جديدة تركّز على اللغة العربية. يبلغ حجم القاموس 65,536
+رمزًا، ويستخدم Unicode NFC مع إمكانية استعادة النص الأصلي بعد فك الترميز.
+
+تتضمن الحزمة ملف `tokenizer.json` الكامل، وملفات إعداد Hugging Face، وتقارير
+التحقق، ونتائج مقارنة على أربع مجموعات بيانات عربية، إضافة إلى سكربت إعادة
+تشغيل المقارنة.
+
+## أبرز النتائج
+
+- المركز الثاني في المتوسط المتساوي لأربعة اختبارات من بين 15 مقطّعًا تم
+  تشغيلها بنجاح.
+- 1.6207 رمز لكل كلمة في المتوسط.
+- المركز الأول على MLQA Arabic بمعدل 1.4397 رمز لكل كلمة.
+- يتفوق في كفاءة التقطيع على Fanar-1 بنسبة 36.81%، وعلى `o200k_base` بنسبة
+  26.37%، وعلى `cl100k_base` بنسبة 164.63% وفق المتوسط المتساوي.
+- الضعف الأوضح هو النص العربي المشكول بالكامل، ويجب أخذه في الاعتبار عند
+  تدريب نماذج للقرآن أو النصوص التراثية والتعليمية المشكولة.
+
+## الاستخدام
+
+```python
+from transformers import PreTrainedTokenizerFast
+
+tokenizer = PreTrainedTokenizerFast.from_pretrained(
+    "SamAI/SamAI-Tokenizer"
+)
+
+text = "المملكة العربية السعودية"
+ids = tokenizer.encode(text, add_special_tokens=False)
+
+print(ids)
+print(tokenizer.convert_ids_to_tokens(ids))
+print(tokenizer.decode(ids))
+```
+
+التفاصيل الكاملة والمنهجية والقيود موجودة في [`README.md`](README.md).

SHA256SUMS

@@ -0,0 +1,6 @@
+7d9496a251b44688557a79fab59b5cf7a12f11a9559540b9a1e223657f35f613  tokenizer.json
+5b31993809ba4357d5244f8acf545bd51cf9e067194a3c5265566115a184f5d0  tokenizer_config.json
+65e01cbbbc962caacbd445bca5bbb15bd5f6b5d6a3e70c7782f2d8082de1133f  special_tokens_map.json
+7accea76f23ec2adf4f0d99070ae3135d8668cdbb319e7bff6809adf98f35d6c  meta.json
+a90011eaa38b558f0660ba749aeb31382c3f2b27ac3a91943e23b4f70a6606e3  reports/validation.json
+5017e97f06b75ce480dcc6d519f44c1822828c2a3223ba5788678520d5584881  reports/stress.json

benchmark/README.md

@@ -0,0 +1,49 @@
+# Benchmark reproduction
+
+The benchmark compares SunAI-64K with Arabic-focused and recent multilingual
+tokenizers on four datasets. It downloads tokenizer assets only, not model
+weights.
+
+## Run
+
+From the repository root:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -r benchmark/requirements.txt
+python benchmark/benchmark.py
+```
+
+Generated files are written to `results/generated/`. Override paths when
+needed:
+
+```bash
+SUNAI_TOKENIZER_PATH=/path/to/tokenizer.json \
+SUNAI_BENCHMARK_OUTPUT=/path/to/results \
+python benchmark/benchmark.py
+```
+
+Third-party remote code is disabled by default. Some custom tokenizers will be
+skipped unless you review their repositories and explicitly opt in:
+
+```bash
+SUNAI_TRUST_REMOTE_CODE=1 python benchmark/benchmark.py
+```
+
+Gated repositories also require accepting their terms and authenticating with
+Hugging Face before running the script.
+
+## Published files
+
+- `arabic_4bench_macro_average.csv`: equal 25% weight per benchmark; primary
+  ranking.
+- `arabic_4bench_pooled_overall.csv`: all tokens and words pooled together.
+- `arabic_4bench_by_benchmark.csv`: result for every tokenizer and benchmark.
+- `arabic_4bench_by_category.csv`: detailed subject, dialect, and intent
+  breakdowns.
+- `arabicmmlu_summary.csv` and `arabicmmlu_by_subject.csv`: the original
+  ArabicMMLU-only comparison.
+
+The script also produces document-level results locally. Those results are not
+published because they contain copies of benchmark text.

benchmark/benchmark.py

@@ -0,0 +1,678 @@
+from pathlib import Path
+import math
+import os
+import random
+
+import pandas as pd
+import tiktoken
+from datasets import get_dataset_config_names, load_dataset
+from huggingface_hub import hf_hub_download
+from tokenizers import Tokenizer as CoreTokenizer
+from transformers import AutoTokenizer
+
+
+# ---------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+MY_TOKENIZER_PATH = Path(
+    os.environ.get("SUNAI_TOKENIZER_PATH", REPO_ROOT / "tokenizer.json")
+)
+
+# Four complementary Arabic benchmarks. Each loader uses evaluation/test data
+# where available, and no prompt/chat-template text is injected.
+EXPECTED_BENCHMARKS = {
+    "ArabicMMLU",
+    "MLQA-Arabic",
+    "MASSIVE-Arabic",
+    "Arabic-Dialects",
+}
+
+# None = use every available example. Set (for example) 1000 for a quick run;
+# the limit is applied independently to each benchmark.
+MAX_EXAMPLES_PER_BENCHMARK = None
+
+BATCH_SIZE = 128
+RANDOM_SEED = 42
+
+OUTPUT_DIR = Path(
+    os.environ.get(
+        "SUNAI_BENCHMARK_OUTPUT",
+        REPO_ROOT / "results" / "generated",
+    )
+)
+
+# Some third-party tokenizers require executing code from their repositories.
+# Keep this disabled by default. Review those repositories and enable it only
+# when you explicitly accept that risk:
+#   SUNAI_TRUST_REMOTE_CODE=1 python benchmark/benchmark.py
+TRUST_REMOTE_CODE = os.environ.get(
+    "SUNAI_TRUST_REMOTE_CODE", "0"
+).lower() in {"1", "true", "yes"}
+
+# Only tokenizer/configuration files are downloaded, not model weights.
+#
+# The list deliberately keeps older baselines while adding newer model
+# families. Some repositories (for example Jais, Gemma, Llama and Aya) may
+# require accepting their Hugging Face terms and running `hf auth login`.
+# A failed or gated tokenizer is reported and skipped without stopping the
+# rest of the benchmark.
+HF_TOKENIZERS = {
+    # Existing Arabic and multilingual baselines
+    "Fanar-1": "QCRI/Fanar-1-9B-Instruct",
+    "ALLaM-7B": "humain-ai/ALLaM-7B-Instruct-preview",
+    "Jais-13B": "inception42/jais-13b",
+    "Qwen2.5-7B": "Qwen/Qwen2.5-7B",
+
+    # Newer Qwen tokenizer generations
+    "Qwen3-8B": "Qwen/Qwen3-8B",
+    "Qwen3.5-4B": "Qwen/Qwen3.5-4B",
+    "Qwen3.8-Flash-Next": "Qwen/Qwen3.8-Flash-Next",
+
+    # Recent frontier/open-weight model families
+    "DeepSeek-V3": "deepseek-ai/DeepSeek-V3",
+    "GPT-OSS-20B": "openai/gpt-oss-20b",
+    "GLM-5.3-Flash": "zai-org/GLM-5.3-Flash",
+    "Kimi-K2.6": "moonshotai/Kimi-K2.6",
+
+    # Recent multilingual tokenizers with Arabic coverage/relevance
+    "Gemma-3-4B": "google/gemma-3-4b-it",
+    "Llama-4-Scout": "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+    "Mistral-Small-3.1": "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
+    "Falcon-H1-7B": "tiiuae/Falcon-H1-7B-Instruct",
+    "Aya-Expanse-8B": "CohereLabs/aya-expanse-8b",
+
+    # Additional Arabic-focused baseline
+    "AceGPT-v2-8B": "FreedomIntelligence/AceGPT-v2-8B",
+}
+
+TIKTOKEN_ENCODINGS = {
+    "tiktoken-cl100k": "cl100k_base",
+    "tiktoken-o200k": "o200k_base",
+}
+
+
+# ---------------------------------------------------------------------
+# Dataset preparation
+# ---------------------------------------------------------------------
+
+def clean_value(value) -> str:
+    """Convert dataset values to clean text while ignoring null/NaN values."""
+    if value is None:
+        return ""
+
+    if isinstance(value, float) and math.isnan(value):
+        return ""
+
+    value = str(value).strip()
+
+    if value.lower() in {"nan", "none", "null"}:
+        return ""
+
+    return value
+
+
+def arabic_mmlu_row_to_text(row) -> str:
+    """
+    Build one benchmark document from the original dataset text.
+
+    No artificial English prompt or chat template is added because we want
+    to measure the tokenizers, not differences in prompt formatting.
+    """
+    parts = []
+
+    context = clean_value(row.get("Context"))
+    question = clean_value(row.get("Question"))
+
+    if context:
+        parts.append(context)
+
+    if question:
+        parts.append(question)
+
+    for index in range(1, 6):
+        option = clean_value(row.get(f"Option {index}"))
+        if option:
+            parts.append(option)
+
+    return "\n".join(parts)
+
+
+def make_record(benchmark, category, text, **metadata):
+    text = clean_value(text)
+    if not text:
+        return None
+
+    return {
+        "benchmark": benchmark,
+        "category": category,
+        "configuration": metadata.get("configuration", ""),
+        "group": metadata.get("group", category),
+        "subject": metadata.get("subject", category),
+        "country": metadata.get("country", ""),
+        "text": text,
+        "words": len(text.split()),
+        "characters": len(text),
+        "bytes": len(text.encode("utf-8")),
+    }
+
+
+def deduplicate_and_limit(records):
+    unique = []
+    seen = set()
+
+    for record in records:
+        if record is None or record["text"] in seen:
+            continue
+        seen.add(record["text"])
+        unique.append(record)
+
+    if (
+        MAX_EXAMPLES_PER_BENCHMARK is not None
+        and len(unique) > MAX_EXAMPLES_PER_BENCHMARK
+    ):
+        random.Random(RANDOM_SEED).shuffle(unique)
+        unique = unique[:MAX_EXAMPLES_PER_BENCHMARK]
+
+    return unique
+
+
+def load_arabic_mmlu():
+    dataset_id = "MBZUAI/ArabicMMLU"
+    print(f"\nLoading ArabicMMLU: {dataset_id}")
+
+    configurations = get_dataset_config_names(dataset_id)
+
+    # Some dataset versions may provide an aggregate "All" configuration.
+    all_config = next(
+        (name for name in configurations if name.lower() == "all"),
+        None,
+    )
+
+    configs_to_load = [all_config] if all_config else configurations
+    records = []
+
+    for position, config_name in enumerate(configs_to_load, start=1):
+        print(
+            f"[{position}/{len(configs_to_load)}] "
+            f"Loading configuration: {config_name}"
+        )
+
+        dataset = load_dataset(
+            dataset_id,
+            config_name,
+            split="test",
+        )
+
+        for row in dataset:
+            # Keep held-out test questions only.
+            if row.get("is_few_shot", 0) not in {0, False, None}:
+                continue
+
+            records.append(
+                make_record(
+                    "ArabicMMLU",
+                    "academic-exams",
+                    arabic_mmlu_row_to_text(row),
+                    configuration=config_name,
+                    group=clean_value(row.get("Group")) or "Unknown",
+                    subject=clean_value(row.get("Subject")) or config_name,
+                    country=clean_value(row.get("Country")) or "Unknown",
+                )
+            )
+
+    return deduplicate_and_limit(records)
+
+
+def load_mlqa_arabic():
+    """Arabic questions and long answer-bearing passages from MLQA."""
+    dataset_id = "mteb/MLQARetrieval"
+    print(f"\nLoading MLQA Arabic: {dataset_id}")
+    records = []
+
+    for config_name in ("ara-ara-corpus", "ara-ara-queries"):
+        category = "passages" if config_name.endswith("corpus") else "questions"
+
+        # Both validation and test exist in the MTEB parquet conversion.
+        for split_name in ("validation", "test"):
+            dataset = load_dataset(dataset_id, config_name, split=split_name)
+            for row in dataset:
+                records.append(
+                    make_record(
+                        "MLQA-Arabic",
+                        category,
+                        row.get("text"),
+                        configuration=config_name,
+                        group="QA",
+                        subject=category,
+                    )
+                )
+
+    return deduplicate_and_limit(records)
+
+
+def load_massive_arabic():
+    """Short Arabic assistant commands from the MASSIVE NLU benchmark."""
+    dataset_id = "mteb/amazon_massive_intent"
+    print(f"\nLoading MASSIVE Arabic: {dataset_id}")
+    configurations = get_dataset_config_names(dataset_id)
+    config_name = "ar" if "ar" in configurations else "default"
+    dataset = load_dataset(dataset_id, config_name, split="test")
+    records = []
+
+    for row in dataset:
+        language = clean_value(row.get("lang"))
+        if config_name == "default" and language not in {"ar", "ar-SA"}:
+            continue
+
+        intent = clean_value(row.get("label_text") or row.get("label"))
+        records.append(
+            make_record(
+                "MASSIVE-Arabic",
+                "assistant-commands",
+                row.get("text"),
+                configuration=config_name,
+                group="spoken-NLU",
+                subject=intent or "assistant-command",
+                country="Saudi Arabia" if language == "ar-SA" else "",
+            )
+        )
+
+    return deduplicate_and_limit(records)
+
+
+def load_arabic_dialects():
+    """Regional dialectal/transcribed Arabic covering five dialect groups."""
+    dataset_id = "arbml/Arabic_Dialects_Dataset"
+    print(f"\nLoading Arabic dialects: {dataset_id}")
+    dataset = load_dataset(dataset_id, split="train")
+    label_feature = dataset.features.get("label")
+    records = []
+
+    for row in dataset:
+        label = row.get("label")
+        if label_feature is not None and hasattr(label_feature, "int2str"):
+            try:
+                dialect = label_feature.int2str(label)
+            except (TypeError, ValueError):
+                dialect = clean_value(label)
+        else:
+            dialect = clean_value(label)
+
+        records.append(
+            make_record(
+                "Arabic-Dialects",
+                dialect or "dialectal-Arabic",
+                row.get("Text") or row.get("text"),
+                configuration="default",
+                group="dialectal-Arabic",
+                subject=dialect or "Unknown dialect",
+            )
+        )
+
+    return deduplicate_and_limit(records)
+
+
+def load_all_benchmarks() -> pd.DataFrame:
+    records = []
+    loaders = (
+        load_arabic_mmlu,
+        load_mlqa_arabic,
+        load_massive_arabic,
+        load_arabic_dialects,
+    )
+
+    for loader in loaders:
+        loaded = loader()
+        records.extend(loaded)
+        print(f"Loaded {len(loaded):,} unique documents")
+
+    frame = pd.DataFrame(records)
+    loaded_names = set(frame["benchmark"].unique())
+    missing = EXPECTED_BENCHMARKS - loaded_names
+    if missing:
+        raise RuntimeError(f"Missing required benchmarks: {sorted(missing)}")
+
+    print("\nBenchmark corpus summary:")
+    print(
+        frame.groupby("benchmark")
+        .agg(documents=("text", "size"), words=("words", "sum"), bytes=("bytes", "sum"))
+        .to_string()
+    )
+    print(f"\nCombined documents: {len(frame):,}")
+    print(f"Combined words: {frame['words'].sum():,}")
+    print(f"Combined UTF-8 bytes: {frame['bytes'].sum():,}")
+
+    return frame
+
+
+# ---------------------------------------------------------------------
+# Tokenizer wrappers
+# ---------------------------------------------------------------------
+
+def encode_in_batches(texts, encode_batch):
+    token_lengths = []
+
+    for start in range(0, len(texts), BATCH_SIZE):
+        batch = texts[start : start + BATCH_SIZE]
+        encoded_batch = encode_batch(batch)
+
+        token_lengths.extend(len(token_ids) for token_ids in encoded_batch)
+
+        completed = min(start + BATCH_SIZE, len(texts))
+        print(
+            f"\rProcessed {completed:,}/{len(texts):,}",
+            end="",
+            flush=True,
+        )
+
+    print()
+    return token_lengths
+
+
+def benchmark_custom_tokenizer(frame):
+    if not MY_TOKENIZER_PATH.exists():
+        raise FileNotFoundError(
+            f"Your tokenizer was not found: {MY_TOKENIZER_PATH}"
+        )
+
+    tokenizer = CoreTokenizer.from_file(str(MY_TOKENIZER_PATH))
+
+    print(f"\nBenchmarking your tokenizer: {MY_TOKENIZER_PATH}")
+    print(f"Vocabulary size: {tokenizer.get_vocab_size():,}")
+
+    return encode_in_batches(
+        frame["text"].tolist(),
+        lambda batch: [
+            encoding.ids
+            for encoding in tokenizer.encode_batch(
+                batch,
+                add_special_tokens=False,
+            )
+        ],
+    )
+
+
+def benchmark_huggingface_tokenizer(name, repository, frame):
+    print(f"\nLoading {name}: {repository}")
+
+    load_kwargs = {
+        "use_fast": True,
+        "trust_remote_code": TRUST_REMOTE_CODE,
+        "token": os.environ.get("HF_TOKEN"),
+    }
+    if name == "Mistral-Small-3.1":
+        load_kwargs["fix_mistral_regex"] = True
+
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(repository, **load_kwargs)
+    except Exception as auto_error:
+        # Some new repositories expose tokenizer.json before their tokenizer
+        # class is supported by the installed Transformers version. The raw
+        # Hugging Face Tokenizers file is sufficient for this compression test.
+        try:
+            tokenizer_path = hf_hub_download(
+                repo_id=repository,
+                filename="tokenizer.json",
+                token=os.environ.get("HF_TOKEN"),
+            )
+            core_tokenizer = CoreTokenizer.from_file(tokenizer_path)
+            print(f"AutoTokenizer unavailable; using tokenizer.json fallback")
+            print(f"Vocabulary size: {core_tokenizer.get_vocab_size():,}")
+            print(f"Benchmarking {name}")
+            return encode_in_batches(
+                frame["text"].tolist(),
+                lambda batch: [
+                    encoding.ids
+                    for encoding in core_tokenizer.encode_batch(
+                        batch,
+                        add_special_tokens=False,
+                    )
+                ],
+            )
+        except Exception as fallback_error:
+            raise RuntimeError(
+                f"AutoTokenizer failed: {auto_error}; "
+                f"tokenizer.json fallback failed: {fallback_error}"
+            ) from auto_error
+
+    print(f"Vocabulary size: {len(tokenizer):,}")
+    print(f"Benchmarking {name}")
+
+    def encode_batch(batch):
+        output = tokenizer(
+            batch,
+            add_special_tokens=False,
+            padding=False,
+            truncation=False,
+            return_attention_mask=False,
+            return_token_type_ids=False,
+        )
+        return output["input_ids"]
+
+    return encode_in_batches(frame["text"].tolist(), encode_batch)
+
+
+def benchmark_tiktoken(name, encoding_name, frame):
+    print(f"\nLoading {name}: {encoding_name}")
+
+    tokenizer = tiktoken.get_encoding(encoding_name)
+
+    def encode_batch(batch):
+        return [
+            tokenizer.encode(
+                text,
+                allowed_special=set(),
+                disallowed_special=(),
+            )
+            for text in batch
+        ]
+
+    return encode_in_batches(frame["text"].tolist(), encode_batch)
+
+
+# ---------------------------------------------------------------------
+# Metrics and reports
+# ---------------------------------------------------------------------
+
+def build_summary(frame, token_columns, benchmark_name):
+    total_documents = len(frame)
+    total_words = int(frame["words"].sum())
+    total_characters = int(frame["characters"].sum())
+    total_bytes = int(frame["bytes"].sum())
+
+    custom_total = int(frame[token_columns["SunAI-64K"]].sum())
+    rows = []
+
+    for tokenizer_name, column in token_columns.items():
+        total_tokens = int(frame[column].sum())
+        ratio = total_tokens / custom_total
+
+        rows.append(
+            {
+                "benchmark": benchmark_name,
+                "tokenizer": tokenizer_name,
+                "documents": total_documents,
+                "tokens": total_tokens,
+                "tokens_per_word": total_tokens / total_words,
+                "characters_per_token": total_characters / total_tokens,
+                "bytes_per_token": total_bytes / total_tokens,
+                "token_ratio_vs_yours": ratio,
+                "extra_tokens_vs_yours_pct": (ratio - 1.0) * 100,
+            }
+        )
+
+    return pd.DataFrame(rows).sort_values(
+        "tokens_per_word",
+        ascending=True,
+    )
+
+
+def build_macro_summary(by_benchmark):
+    """Equal-weight average so every benchmark contributes exactly 25%."""
+    macro = (
+        by_benchmark.groupby("tokenizer", as_index=False)
+        .agg(
+            benchmarks=("benchmark", "nunique"),
+            mean_tokens_per_word=("tokens_per_word", "mean"),
+            mean_characters_per_token=("characters_per_token", "mean"),
+            mean_bytes_per_token=("bytes_per_token", "mean"),
+            mean_token_ratio_vs_yours=("token_ratio_vs_yours", "mean"),
+            mean_extra_tokens_vs_yours_pct=("extra_tokens_vs_yours_pct", "mean"),
+        )
+        .sort_values("mean_tokens_per_word")
+    )
+    macro.insert(0, "rank", range(1, len(macro) + 1))
+    return macro
+
+
+def build_subject_report(frame, token_columns):
+    rows = []
+
+    for (benchmark, group, subject), subset in frame.groupby(
+        ["benchmark", "group", "subject"],
+        dropna=False,
+    ):
+        words = int(subset["words"].sum())
+        byte_count = int(subset["bytes"].sum())
+        custom_tokens = int(subset[token_columns["SunAI-64K"]].sum())
+
+        for tokenizer_name, column in token_columns.items():
+            tokens = int(subset[column].sum())
+
+            rows.append(
+                {
+                    "benchmark": benchmark,
+                    "group": group,
+                    "subject": subject,
+                    "tokenizer": tokenizer_name,
+                    "documents": len(subset),
+                    "words": words,
+                    "tokens": tokens,
+                    "tokens_per_word": tokens / words if words else 0,
+                    "bytes_per_token": byte_count / tokens if tokens else 0,
+                    "token_ratio_vs_yours": (
+                        tokens / custom_tokens if custom_tokens else 0
+                    ),
+                }
+            )
+
+    return pd.DataFrame(rows)
+
+
+# ---------------------------------------------------------------------
+# Main benchmark
+# ---------------------------------------------------------------------
+
+def main():
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    frame = load_all_benchmarks()
+    token_columns = {}
+
+    # Your tokenizer
+    column = "tokens__SunAI_64K"
+    frame[column] = benchmark_custom_tokenizer(frame)
+    token_columns["SunAI-64K"] = column
+
+    # Hugging Face competitors
+    for name, repository in HF_TOKENIZERS.items():
+        try:
+            column = f"tokens__{name}"
+            frame[column] = benchmark_huggingface_tokenizer(
+                name,
+                repository,
+                frame,
+            )
+            token_columns[name] = column
+
+        except Exception as error:
+            print(f"\nWARNING: Could not benchmark {name}")
+            print(f"Repository: {repository}")
+            print(f"Reason: {error}")
+
+    # OpenAI tiktoken baselines
+    for name, encoding_name in TIKTOKEN_ENCODINGS.items():
+        try:
+            column = f"tokens__{name}"
+            frame[column] = benchmark_tiktoken(
+                name,
+                encoding_name,
+                frame,
+            )
+            token_columns[name] = column
+
+        except Exception as error:
+            print(f"\nWARNING: Could not benchmark {name}: {error}")
+
+    by_benchmark = pd.concat(
+        [
+            build_summary(subset, token_columns, benchmark_name)
+            for benchmark_name, subset in frame.groupby("benchmark", sort=True)
+        ],
+        ignore_index=True,
+    )
+    macro_summary = build_macro_summary(by_benchmark)
+    overall_summary = build_summary(frame, token_columns, "ALL-POOLED")
+    subject_report = build_subject_report(frame, token_columns)
+
+    document_path = OUTPUT_DIR / "arabic_4bench_document_results.csv"
+    benchmark_path = OUTPUT_DIR / "arabic_4bench_by_benchmark.csv"
+    macro_path = OUTPUT_DIR / "arabic_4bench_macro_average.csv"
+    overall_path = OUTPUT_DIR / "arabic_4bench_pooled_overall.csv"
+    subject_path = OUTPUT_DIR / "arabic_4bench_by_category.csv"
+
+    frame.to_csv(document_path, index=False)
+    by_benchmark.to_csv(benchmark_path, index=False)
+    macro_summary.to_csv(macro_path, index=False)
+    overall_summary.to_csv(overall_path, index=False)
+    subject_report.to_csv(subject_path, index=False)
+
+    print("\n" + "=" * 100)
+    print("Four-benchmark Arabic tokenizer comparison: equal-weight macro average")
+    print("=" * 100)
+
+    print(
+        macro_summary.to_string(
+            index=False,
+            formatters={
+                "mean_tokens_per_word": "{:.4f}".format,
+                "mean_characters_per_token": "{:.4f}".format,
+                "mean_bytes_per_token": "{:.4f}".format,
+                "mean_token_ratio_vs_yours": "{:.4f}".format,
+                "mean_extra_tokens_vs_yours_pct": "{:+.2f}%".format,
+            },
+        )
+    )
+
+    print("\n" + "=" * 100)
+    print("Results by benchmark")
+    print("=" * 100)
+    for benchmark_name in sorted(by_benchmark["benchmark"].unique()):
+        print(f"\n--- {benchmark_name} ---")
+        subset = by_benchmark[by_benchmark["benchmark"] == benchmark_name]
+        print(
+            subset.to_string(
+                index=False,
+                formatters={
+                    "tokens_per_word": "{:.4f}".format,
+                    "characters_per_token": "{:.4f}".format,
+                    "bytes_per_token": "{:.4f}".format,
+                    "token_ratio_vs_yours": "{:.4f}".format,
+                    "extra_tokens_vs_yours_pct": "{:+.2f}%".format,
+                },
+            )
+        )
+
+    print("\nReports saved to:")
+    print(macro_path)
+    print(benchmark_path)
+    print(overall_path)
+    print(subject_path)
+    print(document_path)
+
+
+if __name__ == "__main__":
+    main()

benchmark/requirements.txt

@@ -0,0 +1,6 @@
+datasets>=3.0
+huggingface_hub>=0.32
+pandas>=2.0
+tiktoken>=0.8
+tokenizers>=0.20
+transformers>=4.45

example.py

@@ -0,0 +1,16 @@
+from transformers import PreTrainedTokenizerFast
+
+
+# Replace this with your published Hugging Face repository ID.
+REPO_ID = "SamAI/SamAI-Tokenizer"
+
+tokenizer = PreTrainedTokenizerFast.from_pretrained(REPO_ID)
+
+text = "المملكة العربية السعودية"
+encoding = tokenizer(text, add_special_tokens=False)
+
+print("Text:", text)
+print("Token IDs:", encoding["input_ids"])
+print("Tokens:", tokenizer.convert_ids_to_tokens(encoding["input_ids"]))
+print("Token count:", len(encoding["input_ids"]))
+print("Decoded:", tokenizer.decode(encoding["input_ids"]))

meta.json

@@ -0,0 +1,15 @@
+{
+  "algorithm": "bpe",
+  "requested_vocab": 65536,
+  "actual_vocab": 65536,
+  "reserved": 256,
+  "file": "tokenizer.json",
+  "characters_by_bucket": {
+    "saudi_schoolbooks": 29856611,
+    "aramix_arabic": 200000000,
+    "fineweb2_arabic": 200000000,
+    "fineweb_english": 200000000
+  },
+  "training_sha256": "bf02ef3cfa689f435aa5402ace1cb09221f9bec22b1f19dd6e73eaf1bddbd115",
+  "normalization": "NFC"
+}

reports/stress.json

@@ -0,0 +1,93 @@
+{
+  "tokenizer_sha256": "7d9496a251b44688557a79fab59b5cf7a12f11a9559540b9a1e223657f35f613",
+  "groups": {
+    "msa": {
+      "documents": 1,
+      "tokens": 30,
+      "words": 13,
+      "characters": 65,
+      "bytes": 115,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 2.3076923076923075,
+      "bytes_per_token": 3.8333333333333335
+    },
+    "gulf_saudi": {
+      "documents": 1,
+      "tokens": 15,
+      "words": 9,
+      "characters": 47,
+      "bytes": 86,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.6666666666666667,
+      "bytes_per_token": 5.733333333333333
+    },
+    "other_dialect": {
+      "documents": 1,
+      "tokens": 14,
+      "words": 8,
+      "characters": 47,
+      "bytes": 86,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.75,
+      "bytes_per_token": 6.142857142857143
+    },
+    "english": {
+      "documents": 1,
+      "tokens": 13,
+      "words": 9,
+      "characters": 67,
+      "bytes": 67,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.4444444444444444,
+      "bytes_per_token": 5.153846153846154
+    },
+    "code": {
+      "documents": 1,
+      "tokens": 22,
+      "words": 7,
+      "characters": 49,
+      "bytes": 49,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 3.142857142857143,
+      "bytes_per_token": 2.227272727272727
+    },
+    "arabizi": {
+      "documents": 1,
+      "tokens": 15,
+      "words": 6,
+      "characters": 35,
+      "bytes": 35,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 2.5,
+      "bytes_per_token": 2.3333333333333335
+    },
+    "unicode": {
+      "documents": 1,
+      "tokens": 40,
+      "words": 7,
+      "characters": 57,
+      "bytes": 85,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 5.714285714285714,
+      "bytes_per_token": 2.125
+    },
+    "math": {
+      "documents": 1,
+      "tokens": 22,
+      "words": 14,
+      "characters": 44,
+      "bytes": 68,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.5714285714285714,
+      "bytes_per_token": 3.090909090909091
+    }
+  }
+}

reports/validation.json

@@ -0,0 +1,49 @@
+{
+  "tokenizer_sha256": "7d9496a251b44688557a79fab59b5cf7a12f11a9559540b9a1e223657f35f613",
+  "groups": {
+    "saudi_schoolbooks": {
+      "documents": 229,
+      "tokens": 31833,
+      "words": 18880,
+      "characters": 116742,
+      "bytes": 205534,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.6860699152542373,
+      "bytes_per_token": 6.456633053749254
+    },
+    "aramix_arabic": {
+      "documents": 5703,
+      "tokens": 3293519,
+      "words": 2454958,
+      "characters": 14302541,
+      "bytes": 25621109,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.341578552464034,
+      "bytes_per_token": 7.779250400559402
+    },
+    "fineweb2_arabic": {
+      "documents": 6024,
+      "tokens": 3770575,
+      "words": 2685728,
+      "characters": 15828573,
+      "bytes": 28416402,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.4039303309940545,
+      "bytes_per_token": 7.536357717324281
+    },
+    "fineweb_english": {
+      "documents": 3433,
+      "tokens": 1967458,
+      "words": 1398259,
+      "characters": 8333087,
+      "bytes": 8383641,
+      "roundtrip_failures": 0,
+      "unknown_tokens": 0,
+      "tokens_per_word": 1.4070769435419332,
+      "bytes_per_token": 4.261153732379547
+    }
+  }
+}

requirements.txt

@@ -0,0 +1,3 @@
+tokenizers>=0.20
+transformers>=4.45
+huggingface_hub>=0.32

results/arabic_4bench_by_benchmark.csv

@@ -0,0 +1,61 @@
+benchmark,tokenizer,documents,tokens,tokens_per_word,characters_per_token,bytes_per_token,token_ratio_vs_yours,extra_tokens_vs_yours_pct
+Arabic-Dialects,ALLaM-7B,9807,432591,1.2811703139021422,4.223703220825214,7.658263810389028,0.9822639315897748,-1.7736068410225236
+Arabic-Dialects,SunAI-64K,9807,440402,1.3043035305476331,4.148791331556169,7.522436319544417,1.0,0.0
+Arabic-Dialects,Falcon-H1-7B,9807,545003,1.6140919820051947,3.352524665001844,6.078674796285525,1.237512545356288,23.751254535628807
+Arabic-Dialects,Mistral-Small-3.1,9807,566146,1.676709521313301,3.2273229873566183,5.851663705122,1.2855209558539697,28.552095585396977
+Arabic-Dialects,Qwen3.8-Flash-Next,9807,571810,1.6934841390421527,3.195355100470436,5.7937007047795595,1.298381932870423,29.83819328704229
+Arabic-Dialects,Qwen3.5-4B,9807,571810,1.6934841390421527,3.195355100470436,5.7937007047795595,1.298381932870423,29.83819328704229
+Arabic-Dialects,GPT-OSS-20B,9807,587234,1.7391641715015118,3.1114274718425703,5.641526205907696,1.3334044804519507,33.34044804519507
+Arabic-Dialects,tiktoken-o200k,9807,587234,1.7391641715015118,3.1114274718425703,5.641526205907696,1.3334044804519507,33.34044804519507
+Arabic-Dialects,Fanar-1,9807,632482,1.8731715696291755,2.8888347810688684,5.2379293007548045,1.4361469748093787,43.61469748093787
+Arabic-Dialects,Qwen2.5-7B,9807,695479,2.059744767557226,2.627161999140161,4.763473807260895,1.5791912843265925,57.91912843265925
+Arabic-Dialects,Qwen3-8B,9807,695479,2.059744767557226,2.627161999140161,4.763473807260895,1.5791912843265925,57.91912843265925
+Arabic-Dialects,DeepSeek-V3,9807,698041,2.067332438923984,2.6175196012841653,4.745990565024117,1.5850086965999246,58.50086965999246
+Arabic-Dialects,GLM-5.3-Flash,9807,747927,2.215075832289362,2.4429336018087326,4.429437632282295,1.6982824782812067,69.82824782812067
+Arabic-Dialects,Kimi-K2.6,9807,885844,2.6235336277183974,2.062593413738762,3.7398187491251282,2.0114440897180303,101.14440897180303
+Arabic-Dialects,tiktoken-cl100k,9807,1294421,3.8335835902538995,1.4115469387471309,2.5593651524504004,2.9391805668457454,193.91805668457454
+ArabicMMLU,ALLaM-7B,14435,818785,1.9942835013128217,3.17674725355252,5.800798744481152,0.8029177396294429,-19.70822603705571
+ArabicMMLU,Qwen3.5-4B,14435,984657,2.3982916266812158,2.641603116618274,4.823615736241148,0.9655753009035442,-3.4424699096455846
+ArabicMMLU,Qwen3.8-Flash-Next,14435,984657,2.3982916266812158,2.641603116618274,4.823615736241148,0.9655753009035442,-3.4424699096455846
+ArabicMMLU,Mistral-Small-3.1,14435,1008568,2.4565307404899577,2.578976330797725,4.709258076797995,0.9890229288794836,-1.097707112051638
+ArabicMMLU,SunAI-64K,14435,1019762,2.483795540790031,2.55066672419643,4.6575642159641175,1.0,0.0
+ArabicMMLU,Falcon-H1-7B,14435,1053222,2.5652927909276464,2.4696341322152406,4.509597216921029,1.0328115776034015,3.2811577603401476
+ArabicMMLU,GPT-OSS-20B,14435,1088017,2.650041649820005,2.390654741607898,4.365379401240974,1.0669322841996465,6.6932284199646475
+ArabicMMLU,tiktoken-o200k,14435,1088017,2.650041649820005,2.390654741607898,4.365379401240974,1.0669322841996465,6.6932284199646475
+ArabicMMLU,DeepSeek-V3,14435,1163831,2.834698927821593,2.2349232835351525,4.081010902785714,1.1412770822995955,14.127708229959545
+ArabicMMLU,Fanar-1,14435,1177059,2.8669178646064215,2.209806815121417,4.035147770842413,1.1542487364698821,15.424873646988214
+ArabicMMLU,Qwen3-8B,14435,1308878,3.1879843922779774,1.987253968666293,3.6287621917397956,1.2835132119063075,28.351321190630753
+ArabicMMLU,Qwen2.5-7B,14435,1308878,3.1879843922779774,1.987253968666293,3.6287621917397956,1.2835132119063075,28.351321190630753
+ArabicMMLU,GLM-5.3-Flash,14435,1368588,3.333417769615604,1.900552248010358,3.4704432597684622,1.3420660899307877,34.20660899307877
+ArabicMMLU,Kimi-K2.6,14435,1471044,3.5829659543167236,1.768181645144537,3.228732111344052,1.4425365918714366,44.253659187143654
+ArabicMMLU,tiktoken-cl100k,14435,1960187,4.774352966392736,1.3269514592230232,2.423037699974543,1.9222004742283003,92.22004742283004
+MASSIVE-Arabic,ALLaM-7B,2907,18333,1.1938655899973951,4.306005563737522,7.766813942071674,0.9513258263712314,-4.867417362876858
+MASSIVE-Arabic,SunAI-64K,2907,19271,1.2549492055222715,4.096414301281719,7.388770691712937,1.0,0.0
+MASSIVE-Arabic,Falcon-H1-7B,2907,23860,1.5537900494920551,3.308549874266555,5.967686504610226,1.2381298323906387,23.81298323906387
+MASSIVE-Arabic,Mistral-Small-3.1,2907,24875,1.619887991664496,3.1735477386934674,5.724180904522613,1.2907996471381868,29.079964713818683
+MASSIVE-Arabic,Qwen3.8-Flash-Next,2907,24973,1.6262698619432143,3.16109398150002,5.7017178552837064,1.295885008562088,29.58850085620881
+MASSIVE-Arabic,Qwen3.5-4B,2907,24973,1.6262698619432143,3.16109398150002,5.7017178552837064,1.295885008562088,29.58850085620881
+MASSIVE-Arabic,GPT-OSS-20B,2907,25338,1.650039072675176,3.1155576604309734,5.6195832346672985,1.314825385293965,31.482538529396507
+MASSIVE-Arabic,tiktoken-o200k,2907,25338,1.650039072675176,3.1155576604309734,5.6195832346672985,1.314825385293965,31.482538529396507
+MASSIVE-Arabic,Fanar-1,2907,27249,1.774485543110185,2.897060442585049,5.225476164262909,1.4139899330600385,41.39899330600385
+MASSIVE-Arabic,Qwen2.5-7B,2907,29540,1.9236780411565513,2.6723764387271496,4.8202098849018284,1.5328732292045042,53.28732292045042
+MASSIVE-Arabic,Qwen3-8B,2907,29540,1.9236780411565513,2.6723764387271496,4.8202098849018284,1.5328732292045042,53.28732292045042
+MASSIVE-Arabic,DeepSeek-V3,2907,30202,1.966788226100547,2.6138004105688366,4.714555327461757,1.5672253645373877,56.72253645373877
+MASSIVE-Arabic,GLM-5.3-Flash,2907,32085,2.089411305027351,2.4604020570359983,4.437868162692848,1.6649369518966322,66.49369518966321
+MASSIVE-Arabic,Kimi-K2.6,2907,37595,2.448228705392029,2.0998005053863547,3.7874451389812474,1.950858803383322,95.0858803383322
+MASSIVE-Arabic,tiktoken-cl100k,2907,55108,3.5886949726491273,1.4324961893010089,2.583817231617914,2.859633646411707,185.9633646411707
+MLQA-Arabic,SunAI-64K,10933,919527,1.4396672657928193,4.123509151987924,7.390784609913575,1.0,0.0
+MLQA-Arabic,ALLaM-7B,10933,982992,1.5390319206898928,3.8572826635415143,6.913612725230724,1.0690191805134597,6.901918051345968
+MLQA-Arabic,Mistral-Small-3.1,10933,1161531,1.8185634123887597,3.2643795129015065,5.850920896644171,1.2631831365473771,26.318313654737715
+MLQA-Arabic,Falcon-H1-7B,10933,1161685,1.8188045241330937,3.263946766980722,5.850145263130711,1.2633506139569584,26.335061395695835
+MLQA-Arabic,Qwen3.8-Flash-Next,10933,1211872,1.8973803365544193,3.1287776266800456,5.607874428982599,1.3179297617144468,31.792976171444675
+MLQA-Arabic,Qwen3.5-4B,10933,1211872,1.8973803365544193,3.1287776266800456,5.607874428982599,1.3179297617144468,31.792976171444675
+MLQA-Arabic,GPT-OSS-20B,10933,1231681,1.928394508914872,3.0784578149699477,5.517683556050633,1.3394723591585673,33.94723591585673
+MLQA-Arabic,tiktoken-o200k,10933,1231681,1.928394508914872,3.0784578149699477,5.517683556050633,1.3394723591585673,33.94723591585673
+MLQA-Arabic,Fanar-1,10933,1349942,2.1135511062958345,2.8087710434966837,5.034309622191175,1.4680830470448394,46.80830470448394
+MLQA-Arabic,DeepSeek-V3,10933,1431242,2.2408393193759903,2.649222144123775,4.748341650119267,1.5564980691159693,55.64980691159693
+MLQA-Arabic,Qwen3-8B,10933,1484612,2.324398629733775,2.553985822558352,4.577644529345041,1.6145387791766854,61.45387791766854
+MLQA-Arabic,Qwen2.5-7B,10933,1484612,2.324398629733775,2.553985822558352,4.577644529345041,1.6145387791766854,61.45387791766854
+MLQA-Arabic,GLM-5.3-Flash,10933,1566196,2.4521314904463387,2.420947314384662,4.339192540397243,1.7032626556914587,70.32626556914587
+MLQA-Arabic,Kimi-K2.6,10933,1802615,2.8222834221584825,2.103431958571298,3.770092892825146,1.9603720173523997,96.03720173523998
+MLQA-Arabic,tiktoken-cl100k,10933,2633608,4.123336485530164,1.4397275524679451,2.5805002111172204,2.8640899070935384,186.40899070935384

results/arabic_4bench_macro_average.csv

@@ -0,0 +1,16 @@
+rank,tokenizer,benchmarks,mean_tokens_per_word,mean_characters_per_token,mean_bytes_per_token,mean_token_ratio_vs_yours,mean_extra_tokens_vs_yours_pct
+1,ALLaM-7B,4,1.502087831475563,3.8909346754141927,7.034872305543145,0.9513816695259771,-4.861833047402281
+2,SunAI-64K,4,1.6206788856631889,3.72984537725556,6.739888959283761,1.0,0.0
+3,Falcon-H1-7B,4,1.8879948366394974,3.0986638596160905,5.601525945236872,1.1929511423268218,19.295114232682163
+4,Mistral-Small-3.1,4,1.8929229164641286,3.0610566424373293,5.534005895771695,1.2071316671047543,20.713166710475434
+5,Qwen3.8-Flash-Next,4,1.9038564910552505,3.0317074563171937,5.481727181321753,1.2194430010126256,21.94430010126255
+6,Qwen3.5-4B,4,1.9038564910552505,3.0317074563171937,5.481727181321753,1.2194430010126256,21.94430010126255
+7,tiktoken-o200k,4,1.9919098507278912,2.924024422212847,5.28604309946665,1.2636586272760324,26.365862727603236
+8,GPT-OSS-20B,4,1.9919098507278912,2.924024422212847,5.28604309946665,1.2636586272760324,26.365862727603236
+9,Fanar-1,4,2.157031520910404,2.7011182705680046,4.883215714512826,1.3681171728460346,36.81171728460347
+10,DeepSeek-V3,4,2.2774147280555286,2.5288663598779824,4.5724746113477135,1.4625023031382192,46.25023031382193
+11,Qwen2.5-7B,4,2.3739514576813825,2.460194557272989,4.44752260331189,1.5025291261535223,50.252912615352244
+12,Qwen3-8B,4,2.3739514576813825,2.460194557272989,4.44752260331189,1.5025291261535223,50.252912615352244
+13,GLM-5.3-Flash,4,2.522509099344664,2.3062088053099377,4.1692353987852115,1.6021370439500213,60.21370439500213
+14,Kimi-K2.6,4,2.869252927396408,2.008501880710238,3.6315222230688935,1.8413028755812972,84.13028755812971
+15,tiktoken-cl100k,4,4.079992003706481,1.402680534934777,2.5366800737900195,2.646276148644823,164.62761486448227

results/arabic_4bench_pooled_overall.csv

@@ -0,0 +1,16 @@
+benchmark,tokenizer,documents,tokens,tokens_per_word,characters_per_token,bytes_per_token,token_ratio_vs_yours,extra_tokens_vs_yours_pct
+ALL-POOLED,ALLaM-7B,38082,2252701,1.6064524778521883,3.6839460718488604,6.659080810103072,0.939031547811095,-6.0968452188905005
+ALL-POOLED,SunAI-64K,38082,2398962,1.7107545338565753,3.459341581900839,6.253086960110248,1.0,0.0
+ALL-POOLED,Mistral-Small-3.1,38082,2761120,1.969017666191489,3.0056024366923566,5.432910558034421,1.1509644587950956,15.096445879509556
+ALL-POOLED,Falcon-H1-7B,38082,2783770,1.9851698979449939,2.98114750859446,5.388705963495547,1.1604060422799527,16.04060422799527
+ALL-POOLED,Qwen3.8-Flash-Next,38082,2793312,1.991974515843093,2.970963859389857,5.37029805478228,1.164383595905229,16.43835959052291
+ALL-POOLED,Qwen3.5-4B,38082,2793312,1.991974515843093,2.970963859389857,5.37029805478228,1.164383595905229,16.43835959052291
+ALL-POOLED,GPT-OSS-20B,38082,2932270,2.0910686359315487,2.830172187417939,5.115803797058252,1.2223078147965662,22.230781479656624
+ALL-POOLED,tiktoken-o200k,38082,2932270,2.0910686359315487,2.830172187417939,5.115803797058252,1.2223078147965662,22.230781479656624
+ALL-POOLED,Fanar-1,38082,3186732,2.2725312936119173,2.6041816506690867,4.70730453643419,1.3283795241441925,32.83795241441925
+ALL-POOLED,DeepSeek-V3,38082,3323316,2.3699324601382177,2.497153144630243,4.513840393149493,1.3853141483691698,38.53141483691698
+ALL-POOLED,Qwen3-8B,38082,3518509,2.509129041712693,2.3586209385850654,4.263430333701008,1.46667975566099,46.66797556609901
+ALL-POOLED,Qwen2.5-7B,38082,3518509,2.509129041712693,2.3586209385850654,4.263430333701008,1.46667975566099,46.66797556609901
+ALL-POOLED,GLM-5.3-Flash,38082,3714796,2.6491057796464763,2.2339931990881867,4.038153912085616,1.548501393519364,54.85013935193639
+ALL-POOLED,Kimi-K2.6,38082,4197098,2.9930463394336235,1.9772778715197978,3.574116687292029,1.7495475126325468,74.95475126325468
+ALL-POOLED,tiktoken-cl100k,38082,5943324,4.238319939698335,1.3963278798194412,2.523994653496932,2.4774564999362223,147.74564999362224

results/arabicmmlu_by_subject.csv

@@ -0,0 +1,295 @@
+group,subject,tokenizer,documents,words,tokens,tokens_per_word,bytes_per_token,token_ratio_vs_yours
+Humanities,History,SunAI-64K,1065,22191,33607,1.5144427921229326,7.207873359716726,1.0
+Humanities,History,Fanar-1,1065,22191,48439,2.1828218647199313,5.000825780879044,1.4413366262980927
+Humanities,History,ALLaM-7B,1065,22191,36070,1.6254337343968275,6.715691710562795,1.0732883030321063
+Humanities,History,Qwen2.5-7B,1065,22191,51053,2.3006173674011987,4.744775037705914,1.5191180408843397
+Humanities,History,Qwen3-8B,1065,22191,51053,2.3006173674011987,4.744775037705914,1.5191180408843397
+Humanities,History,Qwen3.5-4B,1065,22191,44449,2.003019242035059,5.449728902787465,1.3226113607284198
+Humanities,History,Qwen3.8-Flash-Next,1065,22191,44449,2.003019242035059,5.449728902787465,1.3226113607284198
+Humanities,History,DeepSeek-V3,1065,22191,51726,2.3309449776936595,4.6830414105092215,1.5391436307912043
+Humanities,History,GPT-OSS-20B,1065,22191,43976,1.98170429453382,5.5083454611606335,1.308536911952867
+Humanities,History,Kimi-K2.6,1065,22191,63822,2.876030823306746,3.795478048321895,1.8990686464129496
+Humanities,History,Mistral-Small-3.1,1065,22191,42654,1.9221305934838449,5.679068786045858,1.2691998690748951
+Humanities,History,Falcon-H1-7B,1065,22191,42137,1.8988328601685367,5.748748131096186,1.2538161692504537
+Humanities,History,tiktoken-cl100k,1065,22191,92850,4.184128700824659,2.608885298869144,2.762817270211563
+Humanities,History,tiktoken-o200k,1065,22191,43976,1.98170429453382,5.5083454611606335,1.308536911952867
+Humanities,Islamic Studies,SunAI-64K,2210,39174,61563,1.5715270332363303,6.147994737098582,1.0
+Humanities,Islamic Studies,Fanar-1,2210,39174,81872,2.0899576249553276,4.622935802227868,1.3298897064795412
+Humanities,Islamic Studies,ALLaM-7B,2210,39174,62341,1.5913871445346404,6.071269309122408,1.012637460812501
+Humanities,Islamic Studies,Qwen2.5-7B,2210,39174,85998,2.195282585388268,4.401137235749669,1.3969104819453244
+Humanities,Islamic Studies,Qwen3-8B,2210,39174,85998,2.195282585388268,4.401137235749669,1.3969104819453244
+Humanities,Islamic Studies,Qwen3.5-4B,2210,39174,76174,1.9445040077602491,4.968742615590622,1.2373341130224322
+Humanities,Islamic Studies,Qwen3.8-Flash-Next,2210,39174,76174,1.9445040077602491,4.968742615590622,1.2373341130224322
+Humanities,Islamic Studies,DeepSeek-V3,2210,39174,85147,2.173558993209782,4.4451243144209425,1.3830872439614703
+Humanities,Islamic Studies,GPT-OSS-20B,2210,39174,76229,1.945908000204217,4.9651576171798135,1.2382275067816708
+Humanities,Islamic Studies,Kimi-K2.6,2210,39174,108211,2.7623168428039007,3.497694319431481,1.757727856017413
+Humanities,Islamic Studies,Mistral-Small-3.1,2210,39174,73369,1.8729003931178843,5.158704630020853,1.1917710313012686
+Humanities,Islamic Studies,Falcon-H1-7B,2210,39174,75589,1.9295706335834992,5.007196814351294,1.2278316521287136
+Humanities,Islamic Studies,tiktoken-cl100k,2210,39174,153064,3.9072854444274263,2.472749960800711,2.4862985884378603
+Humanities,Islamic Studies,tiktoken-o200k,2210,39174,76229,1.945908000204217,4.9651576171798135,1.2382275067816708
+Humanities,Law,SunAI-64K,314,11281,15675,1.389504476553497,7.622838915470495,1.0
+Humanities,Law,Fanar-1,314,11281,23510,2.0840351032709865,5.082433007230965,1.4998405103668262
+Humanities,Law,ALLaM-7B,314,11281,15815,1.4019147238719971,7.5553588365475814,1.0089314194577352
+Humanities,Law,Qwen2.5-7B,314,11281,24968,2.213278964630795,4.785645626401794,1.5928548644338119
+Humanities,Law,Qwen3-8B,314,11281,24968,2.213278964630795,4.785645626401794,1.5928548644338119
+Humanities,Law,Qwen3.5-4B,314,11281,20174,1.788316638595869,5.922871022107663,1.2870175438596492
+Humanities,Law,Qwen3.8-Flash-Next,314,11281,20174,1.788316638595869,5.922871022107663,1.2870175438596492
+Humanities,Law,DeepSeek-V3,314,11281,25517,2.2619448630440564,4.682682133479641,1.627878787878788
+Humanities,Law,GPT-OSS-20B,314,11281,21664,1.920397127914192,5.515509601181684,1.38207336523126
+Humanities,Law,Kimi-K2.6,314,11281,32118,2.8470880241113377,3.720281462108475,2.0489952153110047
+Humanities,Law,Mistral-Small-3.1,314,11281,20248,1.7948763407499335,5.901224812327143,1.291738437001595
+Humanities,Law,Falcon-H1-7B,314,11281,19290,1.709954791241911,6.194297563504406,1.230622009569378
+Humanities,Law,tiktoken-cl100k,314,11281,47266,4.189876783973052,2.5279905217280922,3.0153748006379586
+Humanities,Law,tiktoken-o200k,314,11281,21664,1.920397127914192,5.515509601181684,1.38207336523126
+Humanities,Philosophy,SunAI-64K,39,1407,2160,1.535181236673774,6.564814814814815,1.0
+Humanities,Philosophy,Fanar-1,39,1407,2989,2.1243781094527363,4.744061559049849,1.3837962962962962
+Humanities,Philosophy,ALLaM-7B,39,1407,2229,1.584221748400853,6.36159712875729,1.0319444444444446
+Humanities,Philosophy,Qwen2.5-7B,39,1407,3076,2.1862117981520965,4.609882964889467,1.424074074074074
+Humanities,Philosophy,Qwen3-8B,39,1407,3076,2.1862117981520965,4.609882964889467,1.424074074074074
+Humanities,Philosophy,Qwen3.5-4B,39,1407,2724,1.9360341151385927,5.2055800293685754,1.261111111111111
+Humanities,Philosophy,Qwen3.8-Flash-Next,39,1407,2724,1.9360341151385927,5.2055800293685754,1.261111111111111
+Humanities,Philosophy,DeepSeek-V3,39,1407,3109,2.2096659559346126,4.560952074622065,1.4393518518518518
+Humanities,Philosophy,GPT-OSS-20B,39,1407,2701,1.919687277896233,5.249907441688263,1.2504629629629629
+Humanities,Philosophy,Kimi-K2.6,39,1407,3901,2.7725657427149963,3.634965393488849,1.8060185185185185
+Humanities,Philosophy,Mistral-Small-3.1,39,1407,2579,1.8329779673063256,5.498255137650252,1.1939814814814815
+Humanities,Philosophy,Falcon-H1-7B,39,1407,2839,2.017768301350391,4.9947164494540335,1.3143518518518518
+Humanities,Philosophy,tiktoken-cl100k,39,1407,5530,3.9303482587064678,2.5641952983725136,2.560185185185185
+Humanities,Philosophy,tiktoken-o200k,39,1407,2701,1.919687277896233,5.249907441688263,1.2504629629629629
+Language,Arabic Language,SunAI-64K,669,17268,32934,1.9072272411396802,5.384982085382887,1.0
+Language,Arabic Language,Fanar-1,669,17268,41489,2.402652304841325,4.274602906794572,1.2597619481387017
+Language,Arabic Language,ALLaM-7B,669,17268,31908,1.847810979847116,5.558135890685722,0.968846784478047
+Language,Arabic Language,Qwen2.5-7B,669,17268,43938,2.5444753300903407,4.036346670308162,1.3341227910366187
+Language,Arabic Language,Qwen3-8B,669,17268,43938,2.5444753300903407,4.036346670308162,1.3341227910366187
+Language,Arabic Language,Qwen3.5-4B,669,17268,38512,2.2302524901552006,4.605032197756543,1.1693690411125282
+Language,Arabic Language,Qwen3.8-Flash-Next,669,17268,38512,2.2302524901552006,4.605032197756543,1.1693690411125282
+Language,Arabic Language,DeepSeek-V3,669,17268,43402,2.5134352559647906,4.086194184599788,1.3178478168458128
+Language,Arabic Language,GPT-OSS-20B,669,17268,38810,2.2475098447996293,4.569672764751353,1.178417440942491
+Language,Arabic Language,Kimi-K2.6,669,17268,52348,3.0315033588139912,3.3878849239703523,1.5894819942916136
+Language,Arabic Language,Mistral-Small-3.1,669,17268,37280,2.158906648135279,4.7572156652360515,1.1319608914799295
+Language,Arabic Language,Falcon-H1-7B,669,17268,38668,2.239286541579801,4.586453915382228,1.1741057873322402
+Language,Arabic Language,tiktoken-cl100k,669,17268,72894,4.221334259902711,2.43297116360743,2.213335762433959
+Language,Arabic Language,tiktoken-o200k,669,17268,38810,2.2475098447996293,4.569672764751353,1.178417440942491
+Language,Arabic Language (General),SunAI-64K,612,130856,563958,4.309760347251941,3.265697090918118,1.0
+Language,Arabic Language (General),Fanar-1,612,130856,544644,4.162162988323042,3.3815042486468223,0.9657527688232103
+Language,Arabic Language (General),ALLaM-7B,612,130856,340840,2.6046952375129915,5.403462034972421,0.6043712475042468
+Language,Arabic Language (General),Qwen2.5-7B,612,130856,638412,4.878736932200281,2.8848392574074424,1.132020469609439
+Language,Arabic Language (General),Qwen3-8B,612,130856,638412,4.878736932200281,2.8848392574074424,1.132020469609439
+Language,Arabic Language (General),Qwen3.5-4B,612,130856,401647,3.069381610319741,4.585409576070529,0.7121931065788587
+Language,Arabic Language (General),Qwen3.8-Flash-Next,612,130856,401647,3.069381610319741,4.585409576070529,0.7121931065788587
+Language,Arabic Language (General),DeepSeek-V3,612,130856,502855,3.84281194595586,3.662519016416263,0.8916532791449009
+Language,Arabic Language (General),GPT-OSS-20B,612,130856,507338,3.877070978785841,3.6301558329949657,0.8996024526649148
+Language,Arabic Language (General),Kimi-K2.6,612,130856,655978,5.0129760958611,2.8075880593556493,1.163168179190649
+Language,Arabic Language (General),Mistral-Small-3.1,612,130856,500913,3.82797120498869,3.676718312361628,0.8882097603012991
+Language,Arabic Language (General),Falcon-H1-7B,612,130856,485719,3.7118588371950847,3.7917314331949132,0.86126803769075
+Language,Arabic Language (General),tiktoken-cl100k,612,130856,800767,6.119451916610625,2.2999399325896297,1.419905383024977
+Language,Arabic Language (General),tiktoken-o200k,612,130856,507338,3.877070978785841,3.6301558329949657,0.8996024526649148
+Language,Arabic Language (Grammar),SunAI-64K,365,9238,18334,1.9846287075124487,5.132158830587978,1.0
+Language,Arabic Language (Grammar),Fanar-1,365,9238,23311,2.523381684347261,4.03642057397795,1.2714628558961492
+Language,Arabic Language (Grammar),ALLaM-7B,365,9238,16880,1.8272353323230137,5.574229857819906,0.9206937929529835
+Language,Arabic Language (Grammar),Qwen2.5-7B,365,9238,24349,2.6357436674604893,3.8643476118115734,1.3280789789462202
+Language,Arabic Language (Grammar),Qwen3-8B,365,9238,24349,2.6357436674604893,3.8643476118115734,1.3280789789462202
+Language,Arabic Language (Grammar),Qwen3.5-4B,365,9238,21007,2.2739770513098074,4.479126005617175,1.1457946983746046
+Language,Arabic Language (Grammar),Qwen3.8-Flash-Next,365,9238,21007,2.2739770513098074,4.479126005617175,1.1457946983746046
+Language,Arabic Language (Grammar),DeepSeek-V3,365,9238,23563,2.550660316085733,3.993252132580741,1.2852078106250682
+Language,Arabic Language (Grammar),GPT-OSS-20B,365,9238,21320,2.307858843905607,4.413367729831145,1.1628668048434603
+Language,Arabic Language (Grammar),Kimi-K2.6,365,9238,28426,3.0770729595150463,3.3101034264405826,1.550452710810516
+Language,Arabic Language (Grammar),Mistral-Small-3.1,365,9238,20445,2.21314137259147,4.602249938860357,1.1151412675902694
+Language,Arabic Language (Grammar),Falcon-H1-7B,365,9238,21652,2.343797358735657,4.345695547755404,1.1809752372640996
+Language,Arabic Language (Grammar),tiktoken-cl100k,365,9238,39172,4.240311755791296,2.402047380782191,2.136576851750845
+Language,Arabic Language (Grammar),tiktoken-o200k,365,9238,21320,2.307858843905607,4.413367729831145,1.1628668048434603
+Other,Driving Test,SunAI-64K,1211,29647,44828,1.5120585556717374,6.785602748282323,1.0
+Other,Driving Test,Fanar-1,1211,29647,63874,2.15448443350086,4.762266336850675,1.4248683858302846
+Other,Driving Test,ALLaM-7B,1211,29647,47040,1.5866698148210612,6.466517857142857,1.0493441599000624
+Other,Driving Test,Qwen2.5-7B,1211,29647,68843,2.3220899247815967,4.4185320221373265,1.5357142857142858
+Other,Driving Test,Qwen3-8B,1211,29647,68843,2.3220899247815967,4.4185320221373265,1.5357142857142858
+Other,Driving Test,Qwen3.5-4B,1211,29647,57867,1.9518669679900158,5.256622945720359,1.2908673150709378
+Other,Driving Test,Qwen3.8-Flash-Next,1211,29647,57867,1.9518669679900158,5.256622945720359,1.2908673150709378
+Other,Driving Test,DeepSeek-V3,1211,29647,67160,2.265321955003879,4.529258487194759,1.4981707861158204
+Other,Driving Test,GPT-OSS-20B,1211,29647,57716,1.9467737039160793,5.270375632406958,1.2874988846256803
+Other,Driving Test,Kimi-K2.6,1211,29647,82860,2.79488649779067,3.6710716871832005,1.8483983224770233
+Other,Driving Test,Mistral-Small-3.1,1211,29647,55718,1.8793807130569704,5.459366811443339,1.2429285268135986
+Other,Driving Test,Falcon-H1-7B,1211,29647,54844,1.849900495834317,5.546367879804537,1.223431783706612
+Other,Driving Test,tiktoken-cl100k,1211,29647,119858,4.042837386582116,2.5378781558177175,2.6737307040242704
+Other,Driving Test,tiktoken-o200k,1211,29647,57716,1.9467737039160793,5.270375632406958,1.2874988846256803
+Other,General Knowledge,SunAI-64K,1198,19003,32538,1.7122559595853286,6.0611592599422215,1.0
+Other,General Knowledge,Fanar-1,1198,19003,44258,2.32900068410251,4.456098332504858,1.3601942344335853
+Other,General Knowledge,ALLaM-7B,1198,19003,34096,1.7942430142609063,5.784197559831065,1.0478824758743623
+Other,General Knowledge,Qwen2.5-7B,1198,19003,46807,2.463137399357996,4.213429615228492,1.4385334070932447
+Other,General Knowledge,Qwen3-8B,1198,19003,46807,2.463137399357996,4.213429615228492,1.4385334070932447
+Other,General Knowledge,Qwen3.5-4B,1198,19003,41421,2.1797084670841445,4.761304652229545,1.2730038723953532
+Other,General Knowledge,Qwen3.8-Flash-Next,1198,19003,41421,2.1797084670841445,4.761304652229545,1.2730038723953532
+Other,General Knowledge,DeepSeek-V3,1198,19003,45788,2.409514287217808,4.307198392591945,1.407216178007253
+Other,General Knowledge,GPT-OSS-20B,1198,19003,40827,2.1484502446982057,4.830577803904279,1.254748294302047
+Other,General Knowledge,Kimi-K2.6,1198,19003,57073,3.0033678892806397,3.4555393969127257,1.7540414284836192
+Other,General Knowledge,Mistral-Small-3.1,1198,19003,39568,2.0821975477556176,4.9842802264456125,1.2160550740672444
+Other,General Knowledge,Falcon-H1-7B,1198,19003,40229,2.1169815292322265,4.902383852444753,1.236369783022927
+Other,General Knowledge,tiktoken-cl100k,1198,19003,80452,4.233647318844393,2.4513747327599065,2.4725551662671337
+Other,General Knowledge,tiktoken-o200k,1198,19003,40827,2.1484502446982057,4.830577803904279,1.254748294302047
+Other,Management,SunAI-64K,75,1686,2805,1.6637010676156583,6.080213903743315,1.0
+Other,Management,Fanar-1,75,1686,3825,2.2686832740213525,4.458823529411765,1.3636363636363635
+Other,Management,ALLaM-7B,75,1686,2906,1.723606168446026,5.868891947694426,1.036007130124777
+Other,Management,Qwen2.5-7B,75,1686,3811,2.260379596678529,4.475203358698504,1.3586452762923351
+Other,Management,Qwen3-8B,75,1686,3811,2.260379596678529,4.475203358698504,1.3586452762923351
+Other,Management,Qwen3.5-4B,75,1686,3362,1.9940688018979833,5.0728732897085065,1.1985739750445632
+Other,Management,Qwen3.8-Flash-Next,75,1686,3362,1.9940688018979833,5.0728732897085065,1.1985739750445632
+Other,Management,DeepSeek-V3,75,1686,3916,2.3226571767497033,4.355209397344229,1.396078431372549
+Other,Management,GPT-OSS-20B,75,1686,3395,2.013641755634638,5.023564064801178,1.2103386809269163
+Other,Management,Kimi-K2.6,75,1686,4812,2.8540925266903914,3.54426433915212,1.7155080213903744
+Other,Management,Mistral-Small-3.1,75,1686,3251,1.9282325029655991,5.246078129806214,1.1590017825311942
+Other,Management,Falcon-H1-7B,75,1686,3347,1.9851720047449586,5.0956080071706005,1.1932263814616755
+Other,Management,tiktoken-cl100k,75,1686,6753,4.005338078291815,2.525544202576633,2.407486631016043
+Other,Management,tiktoken-o200k,75,1686,3395,2.013641755634638,5.023564064801178,1.2103386809269163
+STEM,Biology,SunAI-64K,1409,29117,47477,1.630559466978054,6.448048528761295,1.0
+STEM,Biology,Fanar-1,1409,29117,67911,2.332348799670296,4.507870595338016,1.4303978768666934
+STEM,Biology,ALLaM-7B,1409,29117,51255,1.7603118453137343,5.972763632816311,1.0795753733386693
+STEM,Biology,Qwen2.5-7B,1409,29117,72478,2.489198749871209,4.223819641822346,1.5265918234092297
+STEM,Biology,Qwen3-8B,1409,29117,72478,2.489198749871209,4.223819641822346,1.5265918234092297
+STEM,Biology,Qwen3.5-4B,1409,29117,62891,2.1599409279802178,4.867691720595952,1.324662468142469
+STEM,Biology,Qwen3.8-Flash-Next,1409,29117,62891,2.1599409279802178,4.867691720595952,1.324662468142469
+STEM,Biology,DeepSeek-V3,1409,29117,70678,2.4273791942851255,4.331390248733694,1.4886787286475556
+STEM,Biology,GPT-OSS-20B,1409,29117,63269,2.1729230346532953,4.838609745689042,1.3326242180424206
+STEM,Biology,Kimi-K2.6,1409,29117,86236,2.9617062197341757,3.549955934876386,1.8163742443709585
+STEM,Biology,Mistral-Small-3.1,1409,29117,60698,2.084624102757839,5.043559919601964,1.278471681024496
+STEM,Biology,Falcon-H1-7B,1409,29117,61629,2.1165985506748637,4.967369257979198,1.2980811761484508
+STEM,Biology,tiktoken-cl100k,1409,29117,121687,4.179242367002095,2.51574942269922,2.5630726457021296
+STEM,Biology,tiktoken-o200k,1409,29117,63269,2.1729230346532953,4.838609745689042,1.3326242180424206
+STEM,Computer Science,SunAI-64K,542,9776,14540,1.487315875613748,7.119463548830812,1.0
+STEM,Computer Science,Fanar-1,542,9776,21367,2.1856587561374794,4.844713811016989,1.469532324621733
+STEM,Computer Science,ALLaM-7B,542,9776,15718,1.6078150572831424,6.585888789922382,1.0810178817056395
+STEM,Computer Science,Qwen2.5-7B,542,9776,22331,2.2842675941080195,4.635573865926291,1.5358321870701512
+STEM,Computer Science,Qwen3-8B,542,9776,22331,2.2842675941080195,4.635573865926291,1.5358321870701512
+STEM,Computer Science,Qwen3.5-4B,542,9776,19352,1.9795417348608837,5.349162877221993,1.330949105914718
+STEM,Computer Science,Qwen3.8-Flash-Next,542,9776,19352,1.9795417348608837,5.349162877221993,1.330949105914718
+STEM,Computer Science,DeepSeek-V3,542,9776,22512,2.30278232405892,4.598303127221038,1.5482806052269602
+STEM,Computer Science,GPT-OSS-20B,542,9776,19491,1.9937602291325696,5.311015340413524,1.34050894085282
+STEM,Computer Science,Kimi-K2.6,542,9776,27989,2.863031914893617,3.6984886919861375,1.924965612104539
+STEM,Computer Science,Mistral-Small-3.1,542,9776,18821,1.9252250409165304,5.500079698209447,1.2944291609353507
+STEM,Computer Science,Falcon-H1-7B,542,9776,19322,1.9764729950900164,5.357468170996791,1.3288858321870702
+STEM,Computer Science,tiktoken-cl100k,542,9776,40046,4.096358428805237,2.584952304849423,2.7541953232462175
+STEM,Computer Science,tiktoken-o200k,542,9776,19491,1.9937602291325696,5.311015340413524,1.34050894085282
+STEM,Math,SunAI-64K,409,4585,7744,1.6889858233369683,4.202221074380165,1.0
+STEM,Math,Fanar-1,409,4585,11268,2.4575790621592146,2.8880014199503017,1.4550619834710743
+STEM,Math,ALLaM-7B,409,4585,9977,2.176008724100327,3.2617019144031274,1.2883522727272727
+STEM,Math,Qwen2.5-7B,409,4585,11670,2.5452562704471102,2.788517566409597,1.5069731404958677
+STEM,Math,Qwen3-8B,409,4585,11670,2.5452562704471102,2.788517566409597,1.5069731404958677
+STEM,Math,Qwen3.5-4B,409,4585,10996,2.3982551799345693,2.9594397962895598,1.4199380165289257
+STEM,Math,Qwen3.8-Flash-Next,409,4585,10996,2.3982551799345693,2.9594397962895598,1.4199380165289257
+STEM,Math,DeepSeek-V3,409,4585,9908,2.160959651035987,3.284416633023819,1.2794421487603307
+STEM,Math,GPT-OSS-20B,409,4585,9200,2.0065430752453652,3.5371739130434783,1.18801652892562
+STEM,Math,Kimi-K2.6,409,4585,11418,2.4902944383860413,2.8500613067087057,1.4744318181818181
+STEM,Math,Mistral-Small-3.1,409,4585,10657,2.324318429661941,3.0535798066998217,1.3761621900826446
+STEM,Math,Falcon-H1-7B,409,4585,11069,2.414176663031625,2.939922305537989,1.4293646694214877
+STEM,Math,tiktoken-cl100k,409,4585,14486,3.159432933478735,2.2464448432969766,1.8706095041322315
+STEM,Math,tiktoken-o200k,409,4585,9200,2.0065430752453652,3.5371739130434783,1.18801652892562
+STEM,Natural Science,SunAI-64K,578,9714,17306,1.7815523985999588,6.16751415693979,1.0
+STEM,Natural Science,Fanar-1,578,9714,23515,2.4207329627341982,4.539017648309589,1.3587773026695944
+STEM,Natural Science,ALLaM-7B,578,9714,17754,1.8276714021000617,6.011884645713642,1.0258869756153934
+STEM,Natural Science,Qwen2.5-7B,578,9714,25268,2.601194152769199,4.224117460820009,1.460071651450364
+STEM,Natural Science,Qwen3-8B,578,9714,25268,2.601194152769199,4.224117460820009,1.460071651450364
+STEM,Natural Science,Qwen3.5-4B,578,9714,21571,2.2206094296891083,4.948078438644476,1.2464463191956547
+STEM,Natural Science,Qwen3.8-Flash-Next,578,9714,21571,2.2206094296891083,4.948078438644476,1.2464463191956547
+STEM,Natural Science,DeepSeek-V3,578,9714,24277,2.4991764463660697,4.396548173168019,1.4028082745868484
+STEM,Natural Science,GPT-OSS-20B,578,9714,21630,2.2266831377393452,4.934581599630143,1.2498555414307178
+STEM,Natural Science,Kimi-K2.6,578,9714,29929,3.0810170887379043,3.5662735139830932,1.7294002080203397
+STEM,Natural Science,Mistral-Small-3.1,578,9714,20641,2.1248713197446985,5.171018845986144,1.192707731422628
+STEM,Natural Science,Falcon-H1-7B,578,9714,21462,2.209388511426807,4.973208461466778,1.240147925574945
+STEM,Natural Science,tiktoken-cl100k,578,9714,42341,4.358760551780935,2.5208426820339622,2.446608112793251
+STEM,Natural Science,tiktoken-o200k,578,9714,21630,2.2266831377393452,4.934581599630143,1.2498555414307178
+STEM,Physics,SunAI-64K,255,7398,11407,1.5419032170856988,5.738406241781362,1.0
+STEM,Physics,Fanar-1,255,7398,16104,2.1768045417680453,4.064704421261799,1.411764705882353
+STEM,Physics,ALLaM-7B,255,7398,12596,1.7026223303595567,5.196729120355668,1.1042342421320241
+STEM,Physics,Qwen2.5-7B,255,7398,17613,2.380778588807786,3.7164594333730765,1.5440518979573945
+STEM,Physics,Qwen3-8B,255,7398,17613,2.380778588807786,3.7164594333730765,1.5440518979573945
+STEM,Physics,Qwen3.5-4B,255,7398,15136,2.0459583671262505,4.32465644820296,1.3269045323047253
+STEM,Physics,Qwen3.8-Flash-Next,255,7398,15136,2.0459583671262505,4.32465644820296,1.3269045323047253
+STEM,Physics,DeepSeek-V3,255,7398,16504,2.2308732089753986,3.966190014541929,1.4468308933111247
+STEM,Physics,GPT-OSS-20B,255,7398,14807,2.0014868883482024,4.420746944012967,1.2980625931445604
+STEM,Physics,Kimi-K2.6,255,7398,20342,2.749662070829954,3.2178743486382855,1.7832909616901902
+STEM,Physics,Mistral-Small-3.1,255,7398,14345,1.9390375777237092,4.563123039386546,1.2575611466643288
+STEM,Physics,Falcon-H1-7B,255,7398,14855,2.0079751284130847,4.406462470548637,1.302270535636013
+STEM,Physics,tiktoken-cl100k,255,7398,27688,3.742633144092998,2.3641288644900316,2.4272814938195846
+STEM,Physics,tiktoken-o200k,255,7398,14807,2.0014868883482024,4.420746944012967,1.2980625931445604
+Social Science,Accounting,SunAI-64K,74,2202,4070,1.8483197093551318,5.774447174447174,1.0
+Social Science,Accounting,Fanar-1,74,2202,5447,2.4736603088101727,4.314668624931155,1.3383292383292382
+Social Science,Accounting,ALLaM-7B,74,2202,4242,1.9264305177111716,5.5403111739745405,1.0422604422604422
+Social Science,Accounting,Qwen2.5-7B,74,2202,5808,2.637602179836512,4.046487603305785,1.427027027027027
+Social Science,Accounting,Qwen3-8B,74,2202,5808,2.637602179836512,4.046487603305785,1.427027027027027
+Social Science,Accounting,Qwen3.5-4B,74,2202,4976,2.259763851044505,4.723070739549839,1.2226044226044226
+Social Science,Accounting,Qwen3.8-Flash-Next,74,2202,4976,2.259763851044505,4.723070739549839,1.2226044226044226
+Social Science,Accounting,DeepSeek-V3,74,2202,5669,2.5744777475022707,4.145704709825366,1.392874692874693
+Social Science,Accounting,GPT-OSS-20B,74,2202,5048,2.292461398728429,4.655705229793978,1.2402948402948404
+Social Science,Accounting,Kimi-K2.6,74,2202,7045,3.1993642143505903,3.335982966643009,1.730958230958231
+Social Science,Accounting,Mistral-Small-3.1,74,2202,4963,2.2538601271571297,4.73544227281886,1.2194103194103194
+Social Science,Accounting,Falcon-H1-7B,74,2202,5037,2.287465940054496,4.665872543180464,1.2375921375921375
+Social Science,Accounting,tiktoken-cl100k,74,2202,9647,4.381017257039056,2.436197781693791,2.3702702702702703
+Social Science,Accounting,tiktoken-o200k,74,2202,5048,2.292461398728429,4.655705229793978,1.2402948402948404
+Social Science,Civics,SunAI-64K,323,7045,10769,1.5286018452803407,7.218590398365679,1.0
+Social Science,Civics,Fanar-1,323,7045,15673,2.2246983676366217,4.9599310916863395,1.455381186739716
+Social Science,Civics,ALLaM-7B,323,7045,11696,1.6601845280340668,6.6464603283173735,1.0860804160089146
+Social Science,Civics,Qwen2.5-7B,323,7045,16640,2.361958836053939,4.6716947115384615,1.5451759680564583
+Social Science,Civics,Qwen3-8B,323,7045,16640,2.361958836053939,4.6716947115384615,1.5451759680564583
+Social Science,Civics,Qwen3.5-4B,323,7045,14381,2.041305890702626,5.405535081009665,1.3354071872968707
+Social Science,Civics,Qwen3.8-Flash-Next,323,7045,14381,2.041305890702626,5.405535081009665,1.3354071872968707
+Social Science,Civics,DeepSeek-V3,323,7045,16980,2.410220014194464,4.578150765606596,1.5767480731729966
+Social Science,Civics,GPT-OSS-20B,323,7045,14008,1.988360539389638,5.549471730439748,1.3007707308013743
+Social Science,Civics,Kimi-K2.6,323,7045,20874,2.9629524485450673,3.724106544026061,1.9383415358900549
+Social Science,Civics,Mistral-Small-3.1,323,7045,13880,1.9701916252661462,5.60064841498559,1.2888847618163246
+Social Science,Civics,Falcon-H1-7B,323,7045,13354,1.895528743789922,5.8212520593080725,1.2400408580183861
+Social Science,Civics,tiktoken-cl100k,323,7045,30289,4.299361249112846,2.5665092938030307,2.812610270220076
+Social Science,Civics,tiktoken-o200k,323,7045,14008,1.988360539389638,5.549471730439748,1.3007707308013743
+Social Science,Economics,SunAI-64K,584,15822,24265,1.5336240677537605,7.296723676076653,1.0
+Social Science,Economics,Fanar-1,584,15822,35489,2.2430160535962584,4.989010679365437,1.462559241706161
+Social Science,Economics,ALLaM-7B,584,15822,25510,1.6123119706737454,6.9406115248922,1.0513084689882546
+Social Science,Economics,Qwen2.5-7B,584,15822,37455,2.3672734167614715,4.727139233747163,1.5435812899237584
+Social Science,Economics,Qwen3-8B,584,15822,37455,2.3672734167614715,4.727139233747163,1.5435812899237584
+Social Science,Economics,Qwen3.5-4B,584,15822,31162,1.969536088990014,5.681759835697323,1.2842365547084278
+Social Science,Economics,Qwen3.8-Flash-Next,584,15822,31162,1.969536088990014,5.681759835697323,1.2842365547084278
+Social Science,Economics,DeepSeek-V3,584,15822,37290,2.3568448995070157,4.74805577902923,1.5367813723470019
+Social Science,Economics,GPT-OSS-20B,584,15822,31515,1.9918467956010617,5.618118356338251,1.2987842571605193
+Social Science,Economics,Kimi-K2.6,584,15822,46902,2.9643534319302236,3.7749989339473795,1.9329074799093344
+Social Science,Economics,Mistral-Small-3.1,584,15822,31225,1.9735178864871699,5.670296236989592,1.2868328868740986
+Social Science,Economics,Falcon-H1-7B,584,15822,29581,1.8696119327518645,5.985429836719516,1.2190809808365959
+Social Science,Economics,tiktoken-cl100k,584,15822,67099,4.2408671470104915,2.6387129465416774,2.7652586029260253
+Social Science,Economics,tiktoken-o200k,584,15822,31515,1.9918467956010617,5.618118356338251,1.2987842571605193
+Social Science,Geography,SunAI-64K,1367,27069,43937,1.6231482507665596,6.795479891663063,1.0
+Social Science,Geography,Fanar-1,1367,27069,62704,2.3164505522922902,4.76162605256443,1.4271343059380477
+Social Science,Geography,ALLaM-7B,1367,27069,46879,1.7318334626325316,6.369013844151966,1.0669595102077976
+Social Science,Geography,Qwen2.5-7B,1367,27069,67231,2.4836898296944843,4.441001918757716,1.5301681953706443
+Social Science,Geography,Qwen3-8B,1367,27069,67231,2.4836898296944843,4.441001918757716,1.5301681953706443
+Social Science,Geography,Qwen3.5-4B,1367,27069,56939,2.1034763013040747,5.243734522910483,1.2959237089469013
+Social Science,Geography,Qwen3.8-Flash-Next,1367,27069,56939,2.1034763013040747,5.243734522910483,1.2959237089469013
+Social Science,Geography,DeepSeek-V3,1367,27069,65243,2.4102478850345412,4.576322364085036,1.4849215922798553
+Social Science,Geography,GPT-OSS-20B,1367,27069,56698,2.094573127932321,5.266023492892166,1.290438582515875
+Social Science,Geography,Kimi-K2.6,1367,27069,80426,2.971147807455022,3.7123940019396713,1.8304845574345086
+Social Science,Geography,Mistral-Small-3.1,1367,27069,54771,2.0233846835863902,5.451297219331398,1.2465803309283747
+Social Science,Geography,Falcon-H1-7B,1367,27069,55178,2.0384203332225055,5.411087752365073,1.2558435942372033
+Social Science,Geography,tiktoken-cl100k,1367,27069,115984,4.284753777383723,2.5742602427921093,2.639779684548331
+Social Science,Geography,tiktoken-o200k,1367,27069,56698,2.094573127932321,5.266023492892166,1.290438582515875
+Social Science,Political Science,SunAI-64K,210,3858,10023,2.5979782270606533,4.5571186271575375,1.0
+Social Science,Political Science,Fanar-1,210,3858,12004,3.111456713322965,3.8050649783405532,1.1976454155442482
+Social Science,Political Science,ALLaM-7B,210,3858,12164,3.152928978745464,3.7550147977638932,1.2136086999900229
+Social Science,Political Science,Qwen2.5-7B,210,3858,12639,3.276049766718507,3.6138935042329297,1.2609997006884166
+Social Science,Political Science,Qwen3-8B,210,3858,12639,3.276049766718507,3.6138935042329297,1.2609997006884166
+Social Science,Political Science,Qwen3.5-4B,210,3858,14579,3.7789009849663038,3.132999519857329,1.454554524593435
+Social Science,Political Science,Qwen3.8-Flash-Next,210,3858,14579,3.7789009849663038,3.132999519857329,1.454554524593435
+Social Science,Political Science,DeepSeek-V3,210,3858,14129,3.6622602384655263,3.2327836364923206,1.4096577870896938
+Social Science,Political Science,GPT-OSS-20B,210,3858,13506,3.5007776049766717,3.3819043388123795,1.3475007482789585
+Social Science,Political Science,Kimi-K2.6,210,3858,15329,3.9733022291342666,2.979711657642377,1.529382420433004
+Social Science,Political Science,Mistral-Small-3.1,210,3858,14998,3.8875064800414725,3.045472729697293,1.4963583757358077
+Social Science,Political Science,Falcon-H1-7B,210,3858,12929,3.351218247796786,3.5328331657514114,1.2899331537463834
+Social Science,Political Science,tiktoken-cl100k,210,3858,22315,5.784085018144116,2.046874299798342,2.226379327546643
+Social Science,Political Science,tiktoken-o200k,210,3858,13506,3.5007776049766717,3.3819043388123795,1.3475007482789585
+Social Science,Social Science,SunAI-64K,946,12617,20441,1.6201157168899105,6.404823638765227,1.0
+Social Science,Social Science,Fanar-1,946,12617,28272,2.2407862407862407,4.6307654216185625,1.383102587936011
+Social Science,Social Science,ALLaM-7B,946,12617,21541,1.7072996750416105,6.0777586927255,1.0538134142165256
+Social Science,Social Science,Qwen2.5-7B,946,12617,29450,2.3341523341523343,4.44553480475382,1.4407318624333447
+Social Science,Social Science,Qwen3-8B,946,12617,29450,2.3341523341523343,4.44553480475382,1.4407318624333447
+Social Science,Social Science,Qwen3.5-4B,946,12617,26159,2.0733137829912023,5.004816697886005,1.2797319113546304
+Social Science,Social Science,Qwen3.8-Flash-Next,946,12617,26159,2.0733137829912023,5.004816697886005,1.2797319113546304
+Social Science,Social Science,DeepSeek-V3,946,12617,29395,2.3297931362447493,4.453852696036741,1.4380411917225184
+Social Science,Social Science,GPT-OSS-20B,946,12617,25690,2.036141713561068,5.096185286103542,1.2567878283841298
+Social Science,Social Science,Kimi-K2.6,946,12617,36157,2.8657367044463817,3.6209032829051084,1.7688469252971968
+Social Science,Social Science,Mistral-Small-3.1,946,12617,24923,1.9753507172862013,5.253019299442283,1.219265202289516
+Social Science,Social Science,Falcon-H1-7B,946,12617,25290,2.0044384560513593,5.176789244760775,1.2372193141235752
+Social Science,Social Science,tiktoken-cl100k,946,12617,51644,4.0932075770785445,2.535066997134227,2.526490876180226
+Social Science,Social Science,tiktoken-o200k,946,12617,25690,2.036141713561068,5.096185286103542,1.2567878283841298

results/arabicmmlu_summary.csv

@@ -0,0 +1,15 @@
+tokenizer,documents,tokens,tokens_per_word,characters_per_token,bytes_per_token,token_ratio_vs_yours,extra_tokens_vs_yours_pct
+ALLaM-7B,14455,819457,1.9940358288275573,3.17696962744842,5.801229350655373,0.8030892382355218,-19.69107617644782
+Qwen3.5-4B,14455,985479,2.3980275164616964,2.6417508643005077,4.82390593812755,0.9657951294663464,-3.420487053365362
+Qwen3.8-Flash-Next,14455,985479,2.3980275164616964,2.6417508643005077,4.82390593812755,0.9657951294663464,-3.420487053365362
+SunAI-64K,14455,1020381,2.4829567299503106,2.5513901180049414,4.658904860047374,1.0,0.0
+Falcon-H1-7B,14455,1054020,2.564812606763774,2.4699626193051367,4.510216124931215,1.0329670975841376,3.296709758413763
+Mistral-Small-3.1,14455,1065947,2.593835319768149,2.442325931777096,4.459750813126732,1.0446558687392258,4.465586873922578
+tiktoken-o200k,14455,1088838,2.649537417813186,2.3909801090704037,4.365992002483381,1.0670896459263746,6.708964592637456
+GPT-OSS-20B,14455,1088838,2.649537417813186,2.3909801090704037,4.365992002483381,1.0670896459263746,6.708964592637456
+DeepSeek-V3,14455,1164768,2.834302622677964,2.2351146322701174,4.08137757905437,1.1415030268105737,14.150302681057369
+Fanar-1,14455,1177965,2.8664157058940902,2.210074153306762,4.035653011761809,1.154436431097796,15.443643109779593
+Qwen3-8B,14455,1309838,3.1873105019053227,1.9875664013412346,3.6293480567825944,1.2836754114394524,28.36754114394524
+Qwen2.5-7B,14455,1309838,3.1873105019053227,1.9875664013412346,3.6293480567825944,1.2836754114394524,28.36754114394524
+Kimi-K2.6,14455,1472196,3.582386349810441,1.768371874397159,3.2290931370551204,1.4427904870827661,44.27904870827661
+tiktoken-cl100k,14455,1961832,4.773848167921471,1.3270198467554817,2.4231728302933178,1.9226465408509175,92.26465408509175

special_tokens_map.json

@@ -0,0 +1,11 @@
+{
+  "bos_token": "<bos>",
+  "eos_token": "<eos>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "additional_special_tokens": [
+    "<user>",
+    "<assistant>",
+    "<system>"
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "clean_up_tokenization_spaces": false,
+  "bos_token": "<bos>",
+  "eos_token": "<eos>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "additional_special_tokens": [
+    "<user>",
+    "<assistant>",
+    "<system>"
+  ]
+}

upload_to_hub.py

@@ -0,0 +1,45 @@
+import argparse
+from pathlib import Path
+
+from huggingface_hub import HfApi
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Create a Hugging Face model repository and upload SunAI-64K."
+    )
+    parser.add_argument(
+        "repo_id",
+        help="Destination repository, for example yasseraljohani/SunAI-64K",
+    )
+    parser.add_argument(
+        "--private",
+        action="store_true",
+        help="Create a private repository instead of a public one.",
+    )
+    args = parser.parse_args()
+
+    root = Path(__file__).resolve().parent
+    api = HfApi()
+    api.create_repo(
+        repo_id=args.repo_id,
+        repo_type="model",
+        private=args.private,
+        exist_ok=True,
+    )
+    api.upload_folder(
+        repo_id=args.repo_id,
+        repo_type="model",
+        folder_path=root,
+        ignore_patterns=[
+            "*.zip",
+            "__pycache__/**",
+            "results/generated/**",
+        ],
+        commit_message="Publish SunAI-64K tokenizer and benchmark results",
+    )
+    print(f"Uploaded to https://huggingface.co/{args.repo_id}")
+
+
+if __name__ == "__main__":
+    main()

verify_tokenizer.py

@@ -0,0 +1,36 @@
+from hashlib import sha256
+from pathlib import Path
+
+from tokenizers import Tokenizer
+
+
+ROOT = Path(__file__).resolve().parent
+TOKENIZER_PATH = ROOT / "tokenizer.json"
+EXPECTED_SHA256 = "7d9496a251b44688557a79fab59b5cf7a12f11a9559540b9a1e223657f35f613"
+
+actual_sha256 = sha256(TOKENIZER_PATH.read_bytes()).hexdigest()
+if actual_sha256 != EXPECTED_SHA256:
+    raise RuntimeError(
+        f"SHA-256 mismatch: expected {EXPECTED_SHA256}, got {actual_sha256}"
+    )
+
+tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH))
+if tokenizer.get_vocab_size() != 65_536:
+    raise RuntimeError(f"Unexpected vocabulary size: {tokenizer.get_vocab_size()}")
+
+samples = [
+    "المملكة العربية السعودية",
+    "السلام عليكم ورحمة الله وبركاته",
+    "Artificial intelligence والذكاء الاصطناعي",
+    "ياهلا ومرحبا فيكم",
+]
+
+for text in samples:
+    encoding = tokenizer.encode(text, add_special_tokens=False)
+    decoded = tokenizer.decode(encoding.ids)
+    if decoded != text:
+        raise RuntimeError(f"Round-trip failure: {text!r} -> {decoded!r}")
+
+print("Tokenizer verification passed")
+print(f"Vocabulary size: {tokenizer.get_vocab_size():,}")
+print(f"SHA-256: {actual_sha256}")