Hugging Face's logo

LLMDH
/
1b_rag_675k

Safetensors
llama
Model card Files
xet
 Community
Pclanglais commited on
Commit
291e715
·
verified ·
1 Parent(s): 426ecc3

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +1 -15
  2. tokenizer.json +0 -0
  3. tokenizer_config.json +1 -102
special_tokens_map.json CHANGED
@@ -1,15 +1 @@
1
- {
2
- "additional_special_tokens": [
3
- "<|source_id|>",
4
- "<|source_analysis_start|>",
5
- "<|source_analysis_end|>",
6
- "<|source_start|>",
7
- "<|source_end|>",
8
- "<|answer_start|>",
9
- "<|answer_end|>",
10
- "<|query_start|>",
11
- "<|query_end|>",
12
- "<|source_interpretation_start|>",
13
- "<|source_interpretation_end|>"
14
- ]
15
- }
 
1
+ {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -31,110 +31,9 @@
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
34
- },
35
- "65520": {
36
- "content": "<|source_id|>",
37
- "lstrip": false,
38
- "normalized": false,
39
- "rstrip": false,
40
- "single_word": false,
41
- "special": true
42
- },
43
- "65521": {
44
- "content": "<|source_analysis_start|>",
45
- "lstrip": false,
46
- "normalized": false,
47
- "rstrip": false,
48
- "single_word": false,
49
- "special": true
50
- },
51
- "65522": {
52
- "content": "<|source_analysis_end|>",
53
- "lstrip": false,
54
- "normalized": false,
55
- "rstrip": false,
56
- "single_word": false,
57
- "special": true
58
- },
59
- "65523": {
60
- "content": "<|source_start|>",
61
- "lstrip": false,
62
- "normalized": false,
63
- "rstrip": false,
64
- "single_word": false,
65
- "special": true
66
- },
67
- "65524": {
68
- "content": "<|source_end|>",
69
- "lstrip": false,
70
- "normalized": false,
71
- "rstrip": false,
72
- "single_word": false,
73
- "special": true
74
- },
75
- "65525": {
76
- "content": "<|answer_start|>",
77
- "lstrip": false,
78
- "normalized": false,
79
- "rstrip": false,
80
- "single_word": false,
81
- "special": true
82
- },
83
- "65526": {
84
- "content": "<|answer_end|>",
85
- "lstrip": false,
86
- "normalized": false,
87
- "rstrip": false,
88
- "single_word": false,
89
- "special": true
90
- },
91
- "65527": {
92
- "content": "<|query_start|>",
93
- "lstrip": false,
94
- "normalized": false,
95
- "rstrip": false,
96
- "single_word": false,
97
- "special": true
98
- },
99
- "65528": {
100
- "content": "<|query_end|>",
101
- "lstrip": false,
102
- "normalized": false,
103
- "rstrip": false,
104
- "single_word": false,
105
- "special": true
106
- },
107
- "65529": {
108
- "content": "<|source_interpretation_start|>",
109
- "lstrip": false,
110
- "normalized": false,
111
- "rstrip": false,
112
- "single_word": false,
113
- "special": true
114
- },
115
- "65530": {
116
- "content": "<|source_interpretation_end|>",
117
- "lstrip": false,
118
- "normalized": false,
119
- "rstrip": false,
120
- "single_word": false,
121
- "special": true
122
  }
123
  },
124
- "additional_special_tokens": [
125
- "<|source_id|>",
126
- "<|source_analysis_start|>",
127
- "<|source_analysis_end|>",
128
- "<|source_start|>",
129
- "<|source_end|>",
130
- "<|answer_start|>",
131
- "<|answer_end|>",
132
- "<|query_start|>",
133
- "<|query_end|>",
134
- "<|source_interpretation_start|>",
135
- "<|source_interpretation_end|>"
136
- ],
137
  "clean_up_tokenization_spaces": true,
138
  "model_max_length": 1000000000000000019884624838656,
139
  "tokenizer_class": "PreTrainedTokenizerFast"
140
- }
 
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  }
35
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  "clean_up_tokenization_spaces": true,
37
  "model_max_length": 1000000000000000019884624838656,
38
  "tokenizer_class": "PreTrainedTokenizerFast"
39
+ }