init commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,8 @@
+.mypy_cache/
+__pycache__/
+.ipynb_checkpoints/
+env/
+venv/
+*.pyc
+*.pyo
+*.pyd

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 CyberAgent AI Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,143 @@
+# CAT-Translate 🐱
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Models-blue)](https://huggingface.co/cyberagent/CAT-Translate-0.8b/)
+
+Tiny Language Model For Japanese and English Bidirectional Translation
+
+- **Purrs on your lap** 🐱: Small and efficient! 0.8-3.3B models that run on edge devices.
+- **Swift and Feline Sharp** 🐾: Beats TranslateGemma-12B on text-to-text translation quality.
+- **Adopt and adapt** 🐈: Open source (MIT License) models you can customize and extend.
+
+<div align="center">
+  <img src="CAT-logo.png" alt="Cat sleeping on top of a laptop." width="200">
+</div>
+
+## Models
+
+All models are available on Hugging Face:
+
+- [CAT-Translate-0.8B](https://huggingface.co/cyberagent/CAT-Translate-0.8b/)
+- [CAT-Translate-1.4B](https://huggingface.co/cyberagent/CAT-Translate-1.4b/)
+- [CAT-Translate-3.3B (In preparation)](https://huggingface.co/cyberagent/CAT-Translate-3.3b/)
+
+## Evaluation
+
+We conducted evaluation on the translation subsets of the following benchmarks:
+
+- [The Business Scene Dialogue corpus](https://github.com/tsuruoka-lab/BSD) (BSD)
+  - Each conversation is given to the model to translate instead of each sentence.
+- [Court Interpreter](https://github.com/mynlp/court_interpreter) (Court)
+- [JMedBench](https://huggingface.co/datasets/Coldog2333/JMedBench) (JMed)
+  - ejmmt subsets are used.
+- [pfmt-bench-fin-ja](https://github.com/pfnet-research/pfmt-bench-fin-ja) (PFMT)
+- [WAT 2025 Patent Translation](https://sites.google.com/view/pat-claims-trans-2025/) (wat-pat-2025)
+
+We chose these tasks as benchmarks because (1) they are derived from real world applications and (2) are less overoptimized compared to popular datasets (e.g., WMT).
+
+The results are below.
+Overall, our 1.4B model achieved the best overall scores.
+The 0.8B, 1.4B, and 3.3B-beta models achieved the best scores among all models (including closed source) within their respective sizes for both En-Ja and Ja-En translation tasks.
+
+
+| Model                                            | Avg. BLEU | Avg. BLEU Ja->En | Avg. BLEU En->Ja | BSD (Ja-En) | Court (Ja-En) | JMed (Ja-En) | PFMT (Ja-En) | wat-pat-2025 (Ja-En) | BSD (En-Ja) | JMed (En-Ja) | PFMT (En-Ja) | wat-pat-2025 (En-Ja) |
+|:-------------------------------------------------|----------:|-----------------:|-----------------:|------------:|--------------:|-------------:|-------------:|------------------:|------------:|-------------:|-------------:|------------------:|
+| CyberAgent/CAT-Translate-1.4B                      |   33.73  |   33.26  |   34.19  |  31.28  |  43.84  |  24.08  |  36.55  |  30.57  |  15.71  |  26.92  |  51.53  |  42.58  |
+| Unbabel/Tower-Plus-9B                              |   32.41  |   36.84  |   27.99  |  15.43  |  40.54  |  29.13  |  58.00  |  41.10  |  10.00  |  18.80  |  53.00  |  30.16  |
+| google/translategemma-12b-it                       |   32.24  |   35.81  |   28.68  |  31.58  |  34.30  |  23.46  |  48.75  |  40.97  |  15.92  |  21.79  |  52.53  |  24.47  |
+| CyberAgent/CAT-Translate-3.3B-beta                 |   30.60  |   30.32  |   30.88  |  17.20  |  38.65  |  23.96  |  40.58  |  31.22  |  16.63  |  26.68  |  53.40  |  26.80  |
+| CyberAgent/CAT-Translate-0.8B                    |   30.42  |   29.71  |   30.68  |  29.63  |  33.19  |  22.96  |  32.51  |  30.56  |  14.60  |  26.22  |  50.62  |  32.87  |
+| google/translategemma-4b-it                      |   28.09  |   29.41  |   26.76  |  28.86  |  25.89  |  21.50  |  42.65  |  28.16  |  14.14  |  20.68  |  51.99  |  20.23  |
+| LiquidAI/LFM2.5-1.2B-JP                          |   25.47  |   24.51  |   26.43  |  19.06  |  29.99  |  22.10  |  43.61  |   7.80  |  14.57  |  23.85  |  54.77  |  12.54  |
+| pfnet/plamo-2-translate                          |   25.24  |   25.92  |   24.57  |  25.55  |  28.63  |  22.90  |  29.02  |  23.48  |  17.35  |  24.98  |  32.04  |  23.89  |
+| LiquidAI/LFM2-350M-ENJP-MT                       |   24.95  |   24.91  |   25.00  |  10.94  |  29.56  |  21.48  |  41.40  |  21.17  |   8.11  |  22.84  |  47.53  |  21.52  |
+| mistralai/Ministral-8B-Instruct-2410             |   24.12  |   27.52  |   20.71  |  19.23  |  29.21  |  16.25  |  50.23  |  22.69  |  12.91  |  16.49  |  41.66  |  11.80  |
+| Rakuten/RakutenAI-2.0-mini-instruct              |   18.43  |   17.24  |   19.62  |   0.11  |  30.62  |  18.21  |  29.34  |   7.90  |   5.19  |  20.36  |  45.70  |   7.23  |
+| SakanaAI/TinySwallow-1.5B-Instruct               |   15.74  |   14.99  |   16.49  |   4.96  |  18.93  |  15.83  |  26.67  |   8.58  |   6.30  |  17.58  |  34.07  |   8.00  |
+| llm-jp/llm-jp-3.1-1.8b-instruct4                 |   15.18  |   16.26  |   14.11  |  18.82  |   2.44  |  15.67  |  30.65  |  13.72  |  15.38  |   4.91  |  25.47  |  10.65  |
+| tencent/HY-MT1.5-1.8B                            |   14.49  |    8.95  |   20.04  |   5.50  |   4.59  |   4.00  |  15.67  |  14.98  |   6.33  |  18.13  |  37.75  |  17.96  |
+| shisa-ai/shisa-v2.1-llama3.2-3b                  |   14.27  |   14.26  |   14.28  |  17.08  |   3.70  |   8.26  |  26.86  |  15.42  |  13.18  |   5.54  |  25.97  |  12.41  |
+| google/gemma-2-2b-jpn-it                         |   14.15  |   16.98  |   11.32  |  20.04  |   8.08  |  11.27  |  31.49  |  14.01  |  12.37  |   4.48  |  16.24  |  12.21  |
+| shisa-ai/shisa-v2.1-lfm2-1.2b                    |   13.08  |   14.02  |   12.14  |  20.93  |   4.95  |   7.68  |  26.72  |   9.80  |  12.11  |   5.54  |  17.60  |  13.30  |
+| microsoft/phi-4                                  |   11.92  |   13.48  |   10.36  |   6.10  |  18.66  |   2.81  |  24.86  |  14.98  |   3.24  |   6.97  |  14.36  |  16.87  |
+| tencent/HY-MT1.5-7B                              |   10.56  |   13.46  |    7.67  |   4.99  |  12.32  |   5.72  |  29.53  |  14.76  |   0.82  |   7.80  |  14.30  |   7.74  |
+| tokyotech-llm/Llama-3.1-Swallow-8B-Instruct-v0.5 |   10.35  |   12.42  |    8.28  |  24.25  |   2.30  |   3.69  |  14.11  |  17.74  |   6.82  |   2.37  |  11.21  |  12.71  |
+| Qwen/Qwen2.5-14B-Instruct                        |    8.39  |    9.88  |    6.89  |  10.81  |   4.70  |   4.27  |  11.18  |  18.46  |   4.01  |   3.69  |  13.42  |   6.42  |
+| meta-llama/Llama-3.2-3B-Instruct                 |    6.06  |    9.90  |    2.23  |  18.60  |   0.41  |   2.72  |  16.62  |  11.17  |   1.44  |   1.10  |   4.50  |   1.87  |
+
+A detailed experimental evaluation will be present in a technical report.
+
+## Usage
+
+The model supports English to Japanese and Japanese to English translation with the following prompt format:
+
+```python
+from transformers import pipeline
+
+# Load the model
+chat_pipeline = pipeline("text-generation", model="CyberAgent/CAT-Translate-0.8b")
+
+# Define the prompt template
+prompt = "Translate the following {src_lang} text into {tgt_lang}.\n\n{src_text}"
+
+# Example: Japanese to English
+src_lang = "Japanese"
+tgt_lang = "English"
+src_text = "🐈はとてもかわいいの。おててがまるくてふわふわなの。"
+
+user_input = [{"role": "user", "content": prompt.format(src_lang=src_lang, tgt_lang=tgt_lang, src_text=src_text)}]
+
+response = chat_pipeline(user_input)
+
+print("-" * 20)
+print("Source Text:")
+print(src_text)
+print("Translation:")
+print(response[0]['generated_text'][-1]['content'])
+```
+
+**Important**: You need to apply the chat template to run the model correctly. The template is the same as [sarashina2.2-0.5b-instruct-v0.1](https://huggingface.co/sbintuitions/sarashina2.2-0.5b-instruct-v0.1).
+
+### Why Use Instructions?
+
+Although the model is specialized for machine translation, we require an instruction prompt to invoke the translation capability. This design choice provides better customizability—extending and merging this model is easier this way. Since the model is open source, any extensions are welcome!
+
+## Training
+
+We used the [sarashina2.2 series](https://huggingface.co/collections/sbintuitions/sarashina22) ([MIT LICENSE](https://huggingface.co/sbintuitions/sarashina2.2-0.5b/blob/main/LICENSE)) as our pretrained model. While Qwen-3 showed higher benchmark scores, we found that sarashina generated more natural Japanese text that avoided "translationese" patterns. We hypothesized that naturalness is more difficult to learn than translation accuracy, leading us to choose sarashina as our base model.
+
+Our training process involved:
+- Synthesizing parallel corpora from monolingual data using large language models
+- Two-stage supervised fine-tuning (SFT) approach
+- Reinforcement learning with [Multi-Objective GRPO (Ichihara et al. 2025)](https://arxiv.org/abs/2509.22047)
+- LoRA for efficient training
+
+For detailed information about our training methodology, data preparation, and technical specifications, please see [TRAINING.md](TRAINING.md).
+
+## License
+
+The model is licensed under the [MIT License](LICENSE).
+
+## Citation
+
+```bibtex
+@misc{cat-translate-2026,
+  title={CAT-Translate: Tiny Language Model For Japanese and English Bidirectional Translation},
+  author={Yuu Jinnai},
+  year={2026},
+  url={https://huggingface.co/cyberagent/CAT-Translate-0.8b}
+}
+```
+
+## Acknowledgments
+
+This project stands on the shoulders of giants. In particular, the following resources significantly helped us develop the model:
+
+- [sarashina](https://huggingface.co/sbintuitions) by SB Intuitions
+- [gpt-oss](https://huggingface.co/openai/gpt-oss-20b) by OpenAI
+- [MetricX](https://huggingface.co/google/metricx-24-hybrid-xl-v2p6-bfloat16) by Juraj Juraska et al. 
+- [Duplodocus](https://github.com/allenai/duplodocus) by AllenAI
+- [fastText](https://github.com/facebookresearch/fastText) by Facebook Research
+- [COMET](https://huggingface.co/Unbabel/wmt22-comet-da) by Ricardo Rei et al.
+- [sacrebleu](https://github.com/mjpost/sacrebleu) by Matt Post
+- Mitsuki Sakamoto for deploying the model with UI for internal testing

TRAINING.md

@@ -0,0 +1,121 @@
+# Training Details
+
+This document provides detailed information about the training methodology used to develop the CAT-Translate models.
+The details will be available in a technical report.
+
+## Table of Contents
+
+- [Training Data](#training-data)
+- [Supervised Fine-Tuning](#supervised-fine-tuning)
+- [Reinforcement Learning](#reinforcement-learning)
+- [LoRA Configuration](#lora-configuration)
+
+## Training Data
+
+We synthesized parallel corpora from monolingual data using large language models. For generating translations, we used:
+
+- **DeepSeek-V3**: Used for initial prototyping only, not used for the rest of the development.
+- **gpt-oss-20b**: Generated most of the data, providing sufficiently high quality for many instances.
+- **gpt-oss-120b**: Used for domains where gpt-oss-20b was not satisfactory (e.g., scientific abstracts).
+
+### Data Filtering
+
+The synthesized data were filtered according to several criteria:
+
+- Texts written mostly in languages other than Japanese and English
+- The ratio of Japanese and English text lengths being too large or too small
+- Duplicated content using MinHash ([Duplodocus](https://github.com/allenai/duplodocus))
+- Low quality according to BLEU score and/or COMET score ([comet-qe](https://huggingface.co/Unbabel/wmt22-comet-da))
+- Manually identified low quality texts
+- Rule-based algorithms manually coded to filter low quality texts identified by hand
+
+## Supervised Fine-Tuning
+
+We applied a two-stage fine-tuning approach.
+
+### First Stage: Focus on Diversity
+
+The first stage focused on diversity of prompts. The dataset consisted of:
+
+- Mostly web-crawled data with relatively low quality translation
+- Some portion from targeted domains including:
+  - Scientific abstracts (arXiv and PubMed)
+  - Patents (USPTO)
+- Most instances were sentence-long, with some paragraph-long instances
+
+**Key Finding**: We found that the performance of the models mostly saturated with this corpus at around 100k instances. This led us to prepare a more challenging and higher quality dataset for the second stage.
+
+### Second Stage: Focus on Quality
+
+The second stage focused on the quality of generated translations. Key characteristics:
+
+- Large portion of data instances generated by **gpt-oss-120b**
+- Focus areas:
+  - Scientific abstracts (arXiv and PubMed)
+  - Patents (USPTO)
+  - Underspecified/misspecified text (e.g., input with typo)
+- Most instances were paragraph-long to multiple-paragraph-long
+- Kept some data from the first stage corpus to maintain diversity
+
+## Reinforcement Learning
+
+We used the same corpus as the second stage of SFT. The model was trained with **Multi-Objective GRPO** (Ichihara et al. 2025).
+
+### Primary Reward Model: MetricX-24
+
+We chose [MetricX-24](https://huggingface.co/google/metricx-24-hybrid-xl-v2p6-bfloat16) as our primary reward model for the following reasons:
+
+- Open source
+- Faster than LLM-as-a-Judge models
+- High agreement with human judgments
+
+We also considered using gpt-oss-120b as a judge, which has very high accuracy. However, it requires significantly more computational resources that were not available under our constraints.
+
+### MetricX Limitations
+
+Like all reward models, MetricX has several misspecifications that generation models may exploit:
+
+1. **Language-agnostic**: Being multilingual, it gives scores regardless of output language, even when the task requires generating Japanese text.
+2. **Format-agnostic**: Syntactic characters such as newlines (`\n`) and markdown syntax (e.g., `*`, `#`) are ignored.
+3. **Allows hallucination**: MetricX is relatively tolerant to hallucination as long as the output text contains the information in the input text. This is not ideal for training language model-based machine translation systems.
+
+### Auxiliary Reward Functions
+
+To remedy these problems, we implemented auxiliary reward functions:
+
+#### 1. BLEU Score (Weight: 0.1)
+
+Used to compute lexical overlap with reference text. Expected to be effective for:
+- Avoiding overoptimization to the other reward model
+- Giving reward to accurate translation of technical terms
+
+#### 2. Format Consistency
+
+Texts too different in format are penalized. This addresses the issue where models often generate markdown-formatted text even when the input is plain text.
+
+#### 3. Length Penalty
+
+Texts that are too long or too short are penalized. This suppressed many hallucinations generated by the models.
+
+### Reward Normalization Strategy
+
+- **MetricX and BLEU** (weights 1.0 and 0.1): Applied with normalization to compute advantages
+  - **Rationale**: Translation quality is difficult to learn, and training pedagogically with relative advantage makes sense
+  
+- **Format consistency and length penalty**: Applied absolutely without normalization (as in Dr. GRPO)
+  - **Rationale**: These are easy to learn by themselves and exist to keep the model from getting out of control. The penalties should be large enough to prevent violations regardless of translation quality improvements, and the model should be able to learn them. Thus, we penalize with a large absolute value rather than a relative value.
+
+## LoRA Configuration
+
+We used LoRA (Low-Rank Adaptation) to reduce computational resource requirements.
+
+| Model Size | LoRA Usage |
+|------------|-----------|
+| 0.5B | Not used |
+| 1B | For GRPO |
+| 3B | For the second stage of SFT and GRPO |
+| 7B | For all processes |
+
+---
+
+For the main project overview, see [README.md](README.md).

config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 80,
+  "hidden_act": "silu",
+  "hidden_size": 1280,
+  "initializer_range": 0.02,
+  "intermediate_size": 4480,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 8,
+  "pad_token_id": 3,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vocab_size": 102400
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3375cdd5cb11dbc5036df4902cf20afbc5ffa560f51bb138bd4ffa8e8c0b10f9
+size 1586121792

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<cls>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "<sep>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

test.py

@@ -0,0 +1,22 @@
+from transformers import pipeline
+
+# model_name = "CyberAgent/CAT-Translate-0.8b"
+# chat_pipeline = pipeline("text-generation", model_name)
+
+chat_pipeline = pipeline("text-generation", model=".")
+
+prompt = "Translate the following {src_lang} text into {tgt_lang}.\n\n {src_text}"
+
+src_lang = "Japanese"
+tgt_lang = "English"
+src_text = "🐈はとてもかわいいの。おててがまるくてふわふわなの。"
+
+user_input = [{"role": "user", "content": prompt.format(src_lang=src_lang, tgt_lang=tgt_lang, src_text=src_text)}]
+
+response = chat_pipeline(user_input)
+
+print("-" * 20)
+print("Source Text:")
+print(src_text)
+print("Translation:")
+print(response[0]['generated_text'][-1]['content'])

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:008293028e1a9d9a1038d9b63d989a2319797dfeaa03f171093a57b33a3a8277
+size 1831879

tokenizer_config.json

@@ -0,0 +1,172 @@
+{
+  "add_bos_token": false,
+  "add_dummy_prefix_space": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<cls>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<|available_tools|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<|tool_calls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<|tool_results|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<|code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<|file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102397": {
+      "content": "<|prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102398": {
+      "content": "<|suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102399": {
+      "content": "<|middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "\n{%- set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}\n{%- macro output_available_tools(tools, message) %}\n{%- if tools and (message == user_messages[-1]) %}\n    {{- '<|available_tools|>[' }}\n    {%- for tool in tools %}\n        {%- set tool = tool.function %}\n        {{- \"{\" }}\n        {%- for key, val in tool.items() if key != \"return\" %}\n            {%- if val is string %}\n                {{- \"'\" + key + \"': '\" + val + \"'\" }}\n            {%- else %}\n                {{- \"'\" + key + \"': \" + val|string }}\n            {%- endif %}\n            {%- if not loop.last %}\n                {{- \", \" }}\n            {%- endif %}\n        {%- endfor %}\n        {{- \"}\" }}\n        {%- if not loop.last %}\n            {{- \", \" }}\n        {%- else %}\n            {{- \"]\" }}\n        {%- endif %}\n    {%- endfor %}\n    {{- eos_token -}}\n{%- endif %}\n{%- endmacro %}\n\n{%- macro output_tool_results(tool_results) %}\n{{- '<|tool_results|>[' }}\n{%- for tool_result in tool_results %}\n    {{- \"{'content': \" + tool_result.content|string + \", 'call_id': '\" + tool_result.call_id + \"'}\" }}\n{%- endfor %}\n{{- ']' }}\n{{- eos_token -}}\n{%- endmacro %}\n\n{%- macro output_tool_calls(tool_calls) %}\n{{- '<|tool_calls|>[' }}\n{%- for tool_call in tool_calls %}\n    {{- \"{'id': '\" + tool_call.id + \"', 'name': '\" + tool_call.name + \"', 'arguments': \" + tool_call.arguments|string + '}' }}\n{%- endfor %}\n{{- ']' }}\n{%- endmacro %}\n\n{%- for message in messages %}\n    {%- if message['role'] == 'user' %}\n        {%- if tools is defined %}\n            {{- output_available_tools(tools, message) }}\n        {%- endif %}\n        {{- '<|user|>' + message['content'] + eos_token -}}\n    {%- elif message['role'] == 'system' %}\n        {{- '<|system|>' + message['content'] + eos_token -}}\n    {%- elif message['role'] == 'assistant' %}\n        {% set assistant_content = \"\" %}\n        {%- if message.content is defined %}\n            {% set assistant_content = message.content %}\n        {%- endif %}\n        {%- if message.tool_calls is defined and message.tool_calls -%}\n            {{- '<|assistant|>' + assistant_content + output_tool_calls(message['tool_calls']) + eos_token -}}\n        {%- else %}\n            {{- '<|assistant|>' + assistant_content + eos_token }}\n        {%- endif %}\n    {%- elif message['role'] == 'tool_results' %}\n        {{- output_tool_results(message.tool_results) }}\n    {%- endif %}\n{%- if loop.last and add_generation_prompt -%}\n  {{- '<|assistant|>' -}}\n{%- endif -%}\n{%- endfor %}\n",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<cls>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "extra_special_tokens": {},
+  "keep_accents": true,
+  "legacy": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sep_token": "<sep>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}