Hugging Face's logo

mohamamdmend
/
tool_lemonade

ONNX
Model card Files
xet
 Community
mohamamdmend commited on
Commit
158d560
·
verified ·
1 Parent(s): ff290f1

Add files using upload-large-folder tool

Browse files
Files changed (14) hide show
  1. .gitattributes +2 -0
  2. added_tokens.json +12 -0
  3. genai_config.json +59 -0
  4. merges.txt +0 -0
  5. model_jit.bin +3 -0
  6. model_jit.onnx +3 -0
  7. model_jit.onnx.data +3 -0
  8. model_jit.pb.bin +3 -0
  9. onnx_utils.1.log +0 -0
  10. onnx_utils.log +111 -0
  11. special_tokens_map.json +30 -0
  12. tokenizer.json +3 -0
  13. tokenizer_config.json +114 -0
  14. vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ model_jit.onnx.data filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|/tool_call|>": 200026,
3
+ "<|/tool|>": 200024,
4
+ "<|assistant|>": 200019,
5
+ "<|end|>": 200020,
6
+ "<|system|>": 200022,
7
+ "<|tag|>": 200028,
8
+ "<|tool_call|>": 200025,
9
+ "<|tool_response|>": 200027,
10
+ "<|tool|>": 200023,
11
+ "<|user|>": 200021
12
+ }
genai_config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "bos_token_id": 199999,
4
+ "context_length": 131072,
5
+ "decoder": {
6
+ "session_options": {
7
+ "log_id": "onnxruntime-genai",
8
+ "custom_ops_library": "onnx_custom_ops.dll",
9
+ "custom_allocator": "shared_d3d_xrt",
10
+ "external_data_file": "model_jit.bin",
11
+ "config_entries": {
12
+ "hybrid_opt_max_seq_length": "4096"
13
+ },
14
+ "provider_options": []
15
+ },
16
+ "filename": "model_jit.onnx",
17
+ "head_size": 128,
18
+ "hidden_size": 3072,
19
+ "inputs": {
20
+ "input_ids": "input_ids",
21
+ "attention_mask": "attention_mask",
22
+ "position_ids": "position_ids",
23
+ "past_key_names": "past_key_values.%d.key",
24
+ "past_value_names": "past_key_values.%d.value"
25
+ },
26
+ "outputs": {
27
+ "logits": "logits",
28
+ "present_key_names": "present.%d.key",
29
+ "present_value_names": "present.%d.value"
30
+ },
31
+ "num_attention_heads": 24,
32
+ "num_hidden_layers": 32,
33
+ "num_key_value_heads": 8
34
+ },
35
+ "eos_token_id": [
36
+ 200020,
37
+ 199999
38
+ ],
39
+ "pad_token_id": 199999,
40
+ "type": "phi3",
41
+ "vocab_size": 200064
42
+ },
43
+ "search": {
44
+ "diversity_penalty": 0.0,
45
+ "do_sample": false,
46
+ "early_stopping": true,
47
+ "length_penalty": 1.0,
48
+ "max_length": 131072,
49
+ "min_length": 0,
50
+ "no_repeat_ngram_size": 0,
51
+ "num_beams": 1,
52
+ "num_return_sequences": 1,
53
+ "past_present_share_buffer": true,
54
+ "repetition_penalty": 1.0,
55
+ "temperature": 1.0,
56
+ "top_k": 50,
57
+ "top_p": 1.0
58
+ }
59
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model_jit.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e05daf5344f22f31654de56156044b11eef75a444d2bae0fb5cb81af8f5ccde7
3
+ size 2163563520
model_jit.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a2dcd82d6689560c737504bb343cc64322c3ebd6055f836ed050baa9cb3a7a3
3
+ size 19554817
model_jit.onnx.data ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ab73aa942c0f2e5829f677107f612c1039b36e601d6df4d2467095ae95540aa
3
+ size 3285310208
model_jit.pb.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84bdc04cb8f4f7a7b97270108141df88614e269a441bed43fd8683cccdacdfc7
3
+ size 7903
onnx_utils.1.log ADDED
File without changes
onnx_utils.log ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2025-10-20 14:54:18 [WARNING] ryzenai_onnx_utils\partitioner.py(612): Using deprecated --verbose/-v flag. Use --console-log-level [level] or use --verbose/-v before 'partition' instead.
2
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(675): Loading model...
3
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(683): Loaded model
4
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(346): Starting passes...
5
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass fastgelu_to_gelu (1 of 24)...
6
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass FastGelu (1 of 3)...
7
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
8
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass QuickGelu (2 of 3)...
9
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
10
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass BiasGelu (3 of 3)...
11
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
12
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass normalize_gqa (2 of 24)...
13
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 32 nodes
14
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass normalize_matmulnbits (3 of 24)...
15
+ 2025-10-20 14:54:18 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass normalize_matmulnbits (1 of 3)...
16
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
17
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass normalize_matmulnbits (2 of 3)...
18
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
19
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass normalize_matmulnbits (3 of 3)...
20
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 225 nodes
21
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_ssmlp (4 of 24)...
22
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass SSMLP (1 of 2)...
23
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 32 nodes
24
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass SSGMLP (2 of 2)...
25
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
26
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_gqa (5 of 24)...
27
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 32 nodes
28
+ 2025-10-20 14:54:19 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_qkv_matmul (6 of 24)...
29
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 32 nodes
30
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_qkv_lrn_matmul (7 of 24)...
31
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
32
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_sln (8 of 24)...
33
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
34
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_gqo (9 of 24)...
35
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass GQO_Pre_Merged_QKV (1 of 3)...
36
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 32 nodes
37
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass GQO_Pre_Split_QKV (2 of 3)...
38
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
39
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass GQO_unsplit_QKV (3 of 3)...
40
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
41
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_matmulnbits (10 of 24)...
42
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 33 nodes
43
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_gqa_rotary_dim (11 of 24)...
44
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
45
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_add_npu_weights (12 of 24)...
46
+ 2025-10-20 14:54:20 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass gqo (1 of 6)...
47
+ 2025-10-20 14:54:23 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 32 nodes
48
+ 2025-10-20 14:54:23 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass matmulnbits (2 of 6)...
49
+ 2025-10-20 14:54:34 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 33 nodes
50
+ 2025-10-20 14:54:34 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass matmulnbits (3 of 6)...
51
+ 2025-10-20 14:54:34 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
52
+ 2025-10-20 14:54:34 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass qmoe (4 of 6)...
53
+ 2025-10-20 14:54:34 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
54
+ 2025-10-20 14:54:34 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass ssmlp (5 of 6)...
55
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 32 nodes
56
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass ssmlp (6 of 6)...
57
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
58
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_prune_logits_matmulnbits (13 of 24)...
59
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass hybrid_llm_prune_logits_matmulnbits (1 of 2)...
60
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 1 nodes
61
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass hybrid_llm_prune_logits_matmulnbits (2 of 2)...
62
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
63
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass remove_parallel_casts (14 of 24)...
64
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass CastAvx (1 of 4)...
65
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
66
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass Cast (2 of 4)...
67
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
68
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass SDCastBf2Bfp (3 of 4)...
69
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
70
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass SDCastBfp2Bfp (4 of 4)...
71
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
72
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_simplify_casts (15 of 24)...
73
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass hybrid_llm_simplify_casts (1 of 2)...
74
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
75
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass hybrid_llm_simplify_casts (2 of 2)...
76
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 127 nodes
77
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_add_cast_attributes (16 of 24)...
78
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass hybrid_llm_add_cast_attributes (1 of 2)...
79
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
80
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass hybrid_llm_add_cast_attributes (2 of 2)...
81
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 67 nodes
82
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass simplify_casts (17 of 24)...
83
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass simplify_casts (1 of 6)...
84
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
85
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass simplify_casts (2 of 6)...
86
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
87
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass simplify_casts (3 of 6)...
88
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
89
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass simplify_casts (4 of 6)...
90
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
91
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass simplify_casts (5 of 6)...
92
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
93
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(413): Starting subpass simplify_casts (6 of 6)...
94
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
95
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_add_empty_cast_attributes (18 of 24)...
96
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 97 nodes
97
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass simplify_matmulnbits (19 of 24)...
98
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(244): Replaced 0 nodes
99
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_prune_validate (20 of 24)...
100
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(242): Finished pass
101
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass remove_dangling_nodes (21 of 24)...
102
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(242): Finished pass
103
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass remove_unused_inputs (22 of 24)...
104
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(242): Finished pass
105
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass sort_graph_io (23 of 24)...
106
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(242): Finished pass
107
+ 2025-10-20 14:54:50 [INFO] ryzenai_onnx_utils\partitioner.py(389): Starting pass hybrid_llm_jit (24 of 24)...
108
+ 2025-10-20 14:54:54 [INFO] ryzenai_onnx_utils\partitioner.py(242): Finished pass
109
+ 2025-10-20 14:54:54 [INFO] ryzenai_onnx_utils\partitioner.py(468): Replaced at least 807 node(s) in total across 24 passes
110
+ 2025-10-20 14:54:54 [INFO] ryzenai_onnx_utils\partitioner.py(714): Saving final model...
111
+ 2025-10-20 14:55:00 [INFO] ryzenai_onnx_utils\partitioner.py(737): Saved final model
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<|endoftext|>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<|endoftext|>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02074693f3ee8b43ff47351e37727ee379f48d418aa52fa1a93d1986b2fc29e0
3
+ size 15523653
tokenizer_config.json ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": false,
5
+ "added_tokens_decoder": {
6
+ "199999": {
7
+ "content": "<|endoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "200018": {
15
+ "content": "<|endofprompt|>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "200019": {
23
+ "content": "<|assistant|>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": true,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "200020": {
31
+ "content": "<|end|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": true,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "200021": {
39
+ "content": "<|user|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": true,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "200022": {
47
+ "content": "<|system|>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": true,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "200023": {
55
+ "content": "<|tool|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": true,
59
+ "single_word": false,
60
+ "special": false
61
+ },
62
+ "200024": {
63
+ "content": "<|/tool|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": true,
67
+ "single_word": false,
68
+ "special": false
69
+ },
70
+ "200025": {
71
+ "content": "<|tool_call|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": true,
75
+ "single_word": false,
76
+ "special": false
77
+ },
78
+ "200026": {
79
+ "content": "<|/tool_call|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": true,
83
+ "single_word": false,
84
+ "special": false
85
+ },
86
+ "200027": {
87
+ "content": "<|tool_response|>",
88
+ "lstrip": false,
89
+ "normalized": false,
90
+ "rstrip": true,
91
+ "single_word": false,
92
+ "special": false
93
+ },
94
+ "200028": {
95
+ "content": "<|tag|>",
96
+ "lstrip": false,
97
+ "normalized": false,
98
+ "rstrip": true,
99
+ "single_word": false,
100
+ "special": false
101
+ }
102
+ },
103
+ "bos_token": "<|endoftext|>",
104
+ "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}",
105
+ "clean_up_tokenization_spaces": false,
106
+ "eos_token": "<|endoftext|>",
107
+ "errors": "replace",
108
+ "extra_special_tokens": {},
109
+ "model_max_length": 1000000,
110
+ "pad_token": "<|endoftext|>",
111
+ "padding_side": "left",
112
+ "tokenizer_class": "GPT2Tokenizer",
113
+ "unk_token": "<|endoftext|>"
114
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff