Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +198 -0
tokenizer.json
CHANGED
|
@@ -5609,6 +5609,204 @@
|
|
| 5609 |
"rstrip": false,
|
| 5610 |
"normalized": false,
|
| 5611 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5612 |
}
|
| 5613 |
],
|
| 5614 |
"normalizer": null,
|
|
|
|
| 5609 |
"rstrip": false,
|
| 5610 |
"normalized": false,
|
| 5611 |
"special": false
|
| 5612 |
+
},
|
| 5613 |
+
{
|
| 5614 |
+
"id": 64641,
|
| 5615 |
+
"content": "<|bos|>",
|
| 5616 |
+
"single_word": false,
|
| 5617 |
+
"lstrip": false,
|
| 5618 |
+
"rstrip": false,
|
| 5619 |
+
"normalized": false,
|
| 5620 |
+
"special": true
|
| 5621 |
+
},
|
| 5622 |
+
{
|
| 5623 |
+
"id": 64642,
|
| 5624 |
+
"content": "<|user_start|>",
|
| 5625 |
+
"single_word": false,
|
| 5626 |
+
"lstrip": false,
|
| 5627 |
+
"rstrip": false,
|
| 5628 |
+
"normalized": false,
|
| 5629 |
+
"special": true
|
| 5630 |
+
},
|
| 5631 |
+
{
|
| 5632 |
+
"id": 64643,
|
| 5633 |
+
"content": "<|user_end|>",
|
| 5634 |
+
"single_word": false,
|
| 5635 |
+
"lstrip": false,
|
| 5636 |
+
"rstrip": false,
|
| 5637 |
+
"normalized": false,
|
| 5638 |
+
"special": true
|
| 5639 |
+
},
|
| 5640 |
+
{
|
| 5641 |
+
"id": 64644,
|
| 5642 |
+
"content": "<|assistant_start|>",
|
| 5643 |
+
"single_word": false,
|
| 5644 |
+
"lstrip": false,
|
| 5645 |
+
"rstrip": false,
|
| 5646 |
+
"normalized": false,
|
| 5647 |
+
"special": true
|
| 5648 |
+
},
|
| 5649 |
+
{
|
| 5650 |
+
"id": 64645,
|
| 5651 |
+
"content": "<|assistant_end|>",
|
| 5652 |
+
"single_word": false,
|
| 5653 |
+
"lstrip": false,
|
| 5654 |
+
"rstrip": false,
|
| 5655 |
+
"normalized": false,
|
| 5656 |
+
"special": true
|
| 5657 |
+
},
|
| 5658 |
+
{
|
| 5659 |
+
"id": 64646,
|
| 5660 |
+
"content": "<|python_start|>",
|
| 5661 |
+
"single_word": false,
|
| 5662 |
+
"lstrip": false,
|
| 5663 |
+
"rstrip": false,
|
| 5664 |
+
"normalized": false,
|
| 5665 |
+
"special": true
|
| 5666 |
+
},
|
| 5667 |
+
{
|
| 5668 |
+
"id": 64647,
|
| 5669 |
+
"content": "<|python_end|>",
|
| 5670 |
+
"single_word": false,
|
| 5671 |
+
"lstrip": false,
|
| 5672 |
+
"rstrip": false,
|
| 5673 |
+
"normalized": false,
|
| 5674 |
+
"special": true
|
| 5675 |
+
},
|
| 5676 |
+
{
|
| 5677 |
+
"id": 64648,
|
| 5678 |
+
"content": "<|output_start|>",
|
| 5679 |
+
"single_word": false,
|
| 5680 |
+
"lstrip": false,
|
| 5681 |
+
"rstrip": false,
|
| 5682 |
+
"normalized": false,
|
| 5683 |
+
"special": true
|
| 5684 |
+
},
|
| 5685 |
+
{
|
| 5686 |
+
"id": 64649,
|
| 5687 |
+
"content": "<|output_end|>",
|
| 5688 |
+
"single_word": false,
|
| 5689 |
+
"lstrip": false,
|
| 5690 |
+
"rstrip": false,
|
| 5691 |
+
"normalized": false,
|
| 5692 |
+
"special": true
|
| 5693 |
+
},
|
| 5694 |
+
{
|
| 5695 |
+
"id": 64650,
|
| 5696 |
+
"content": "<|end_of_text|>",
|
| 5697 |
+
"single_word": false,
|
| 5698 |
+
"lstrip": false,
|
| 5699 |
+
"rstrip": false,
|
| 5700 |
+
"normalized": false,
|
| 5701 |
+
"special": true
|
| 5702 |
+
},
|
| 5703 |
+
{
|
| 5704 |
+
"id": 64651,
|
| 5705 |
+
"content": "<|pad|>",
|
| 5706 |
+
"single_word": false,
|
| 5707 |
+
"lstrip": false,
|
| 5708 |
+
"rstrip": false,
|
| 5709 |
+
"normalized": false,
|
| 5710 |
+
"special": true
|
| 5711 |
+
},
|
| 5712 |
+
{
|
| 5713 |
+
"id": 64652,
|
| 5714 |
+
"content": "<|unk|>",
|
| 5715 |
+
"single_word": false,
|
| 5716 |
+
"lstrip": false,
|
| 5717 |
+
"rstrip": false,
|
| 5718 |
+
"normalized": false,
|
| 5719 |
+
"special": true
|
| 5720 |
+
},
|
| 5721 |
+
{
|
| 5722 |
+
"id": 64653,
|
| 5723 |
+
"content": "<|mask|>",
|
| 5724 |
+
"single_word": false,
|
| 5725 |
+
"lstrip": false,
|
| 5726 |
+
"rstrip": false,
|
| 5727 |
+
"normalized": false,
|
| 5728 |
+
"special": true
|
| 5729 |
+
},
|
| 5730 |
+
{
|
| 5731 |
+
"id": 64654,
|
| 5732 |
+
"content": "<think>",
|
| 5733 |
+
"single_word": false,
|
| 5734 |
+
"lstrip": false,
|
| 5735 |
+
"rstrip": false,
|
| 5736 |
+
"normalized": false,
|
| 5737 |
+
"special": true
|
| 5738 |
+
},
|
| 5739 |
+
{
|
| 5740 |
+
"id": 64655,
|
| 5741 |
+
"content": "</think>",
|
| 5742 |
+
"single_word": false,
|
| 5743 |
+
"lstrip": false,
|
| 5744 |
+
"rstrip": false,
|
| 5745 |
+
"normalized": false,
|
| 5746 |
+
"special": true
|
| 5747 |
+
},
|
| 5748 |
+
{
|
| 5749 |
+
"id": 64656,
|
| 5750 |
+
"content": "<tool_call>",
|
| 5751 |
+
"single_word": false,
|
| 5752 |
+
"lstrip": false,
|
| 5753 |
+
"rstrip": false,
|
| 5754 |
+
"normalized": false,
|
| 5755 |
+
"special": true
|
| 5756 |
+
},
|
| 5757 |
+
{
|
| 5758 |
+
"id": 64657,
|
| 5759 |
+
"content": "</tool_call>",
|
| 5760 |
+
"single_word": false,
|
| 5761 |
+
"lstrip": false,
|
| 5762 |
+
"rstrip": false,
|
| 5763 |
+
"normalized": false,
|
| 5764 |
+
"special": true
|
| 5765 |
+
},
|
| 5766 |
+
{
|
| 5767 |
+
"id": 64658,
|
| 5768 |
+
"content": "<tool_response>",
|
| 5769 |
+
"single_word": false,
|
| 5770 |
+
"lstrip": false,
|
| 5771 |
+
"rstrip": false,
|
| 5772 |
+
"normalized": false,
|
| 5773 |
+
"special": true
|
| 5774 |
+
},
|
| 5775 |
+
{
|
| 5776 |
+
"id": 64659,
|
| 5777 |
+
"content": "</tool_response>",
|
| 5778 |
+
"single_word": false,
|
| 5779 |
+
"lstrip": false,
|
| 5780 |
+
"rstrip": false,
|
| 5781 |
+
"normalized": false,
|
| 5782 |
+
"special": true
|
| 5783 |
+
},
|
| 5784 |
+
{
|
| 5785 |
+
"id": 64660,
|
| 5786 |
+
"content": "<|vision_start|>",
|
| 5787 |
+
"single_word": false,
|
| 5788 |
+
"lstrip": false,
|
| 5789 |
+
"rstrip": false,
|
| 5790 |
+
"normalized": false,
|
| 5791 |
+
"special": true
|
| 5792 |
+
},
|
| 5793 |
+
{
|
| 5794 |
+
"id": 64661,
|
| 5795 |
+
"content": "<|vision_end|>",
|
| 5796 |
+
"single_word": false,
|
| 5797 |
+
"lstrip": false,
|
| 5798 |
+
"rstrip": false,
|
| 5799 |
+
"normalized": false,
|
| 5800 |
+
"special": true
|
| 5801 |
+
},
|
| 5802 |
+
{
|
| 5803 |
+
"id": 64662,
|
| 5804 |
+
"content": "<|vision_pad|>",
|
| 5805 |
+
"single_word": false,
|
| 5806 |
+
"lstrip": false,
|
| 5807 |
+
"rstrip": false,
|
| 5808 |
+
"normalized": false,
|
| 5809 |
+
"special": true
|
| 5810 |
}
|
| 5811 |
],
|
| 5812 |
"normalizer": null,
|