MaziyarPanahi commited on
Commit
e28bf50
·
verified ·
1 Parent(s): 0f5996a

Publish onnx-android artifact

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ model.ort filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ pipeline_tag: token-classification
4
+ library_name: openmed
5
+ tags:
6
+ - openmed
7
+ - medical-nlp
8
+ ---
9
+
10
+ # OpenMed-NER-GenomicDetect-BioMed-335M-v1-onnx-android
11
+
12
+ This model card is rendered from the OpenMed model manifest. Update `models.jsonl` and rerun the publish step instead of editing this file directly.
13
+
14
+ ## Manifest Summary
15
+
16
+ | Field | Value |
17
+ |---|---|
18
+ | Repository | `OpenMed/OpenMed-NER-GenomicDetect-BioMed-335M-v1-onnx-android` |
19
+ | Family | NER |
20
+ | Task | token-classification |
21
+ | Languages | en |
22
+ | Tier | Not specified |
23
+ | Parameters | 335M (335,000,000) |
24
+ | Architecture | bert |
25
+ | Base model | OpenMed/OpenMed-NER-GenomicDetect-BioMed-335M |
26
+ | Formats | onnx-android, int8, ort-android |
27
+ | License | apache-2.0 |
28
+ | arXiv | [arXiv:2508.01630](https://arxiv.org/abs/2508.01630) |
29
+ | Reproducibility hash | `sha256:eda50934f79e910ace442a71c0ed9e8ff0ae7f15e33acfaaf5a5b244229a5da5` |
30
+ | Released | 2026-07-09 |
31
+
32
+ ## Benchmark
33
+
34
+ | Dataset | Micro F1 | Recall |
35
+ |---|---:|---:|
36
+ | Not reported | Not reported | Not reported |
37
+
38
+ ## Canonical Labels
39
+
40
+ `Cell-line-name`
41
+
42
+ ## Artifact Format
43
+
44
+ | Field | Value |
45
+ |---|---|
46
+ | Runtime artifacts | onnx-android, ort-android |
47
+ | Quantization | int8 |
config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "transformers_version": "5.12.1",
3
+ "architectures": [
4
+ "BertForTokenClassification"
5
+ ],
6
+ "output_hidden_states": false,
7
+ "return_dict": true,
8
+ "dtype": "bfloat16",
9
+ "chunk_size_feed_forward": 0,
10
+ "is_encoder_decoder": false,
11
+ "id2label": {
12
+ "0": "B-Cell-line-name",
13
+ "1": "I-Cell-line-name",
14
+ "2": "O"
15
+ },
16
+ "label2id": {
17
+ "B-Cell-line-name": 0,
18
+ "I-Cell-line-name": 1,
19
+ "O": 2
20
+ },
21
+ "problem_type": null,
22
+ "vocab_size": 30522,
23
+ "hidden_size": 1024,
24
+ "num_hidden_layers": 24,
25
+ "num_attention_heads": 16,
26
+ "intermediate_size": 4096,
27
+ "hidden_act": "gelu",
28
+ "hidden_dropout_prob": 0.2,
29
+ "attention_probs_dropout_prob": 0.2,
30
+ "max_position_embeddings": 512,
31
+ "type_vocab_size": 2,
32
+ "initializer_range": 0.02,
33
+ "layer_norm_eps": 1e-07,
34
+ "pad_token_id": 0,
35
+ "use_cache": true,
36
+ "classifier_dropout": 0.2,
37
+ "is_decoder": false,
38
+ "add_cross_attention": false,
39
+ "bos_token_id": 0,
40
+ "eos_token_id": null,
41
+ "tie_word_embeddings": true,
42
+ "_name_or_path": "OpenMed/OpenMed-NER-GenomicDetect-BioMed-335M",
43
+ "eos_token_ids": 0,
44
+ "model_type": "bert",
45
+ "output_past": true,
46
+ "position_embedding_type": "absolute",
47
+ "output_attentions": false,
48
+ "max_sequence_length": 512
49
+ }
id2label.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "0": "B-Cell-line-name",
3
+ "1": "I-Cell-line-name",
4
+ "2": "O"
5
+ }
model.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f22dd3e3c2084aab6529a01a9b5d2af6fa8a8d52c7d2a85e3c5e0a3ca6017425
3
+ size 1336862934
model.ort ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c1f2b501a9bc3eae20cd4d691c3469b96dd547801114d51b8183312fb03e7b6
3
+ size 1337027256
model.required_operators_and_types.config ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ # Generated from model/s:
2
+ # - /Users/maziyar/Developer/openmed/dist/onnx-android/OpenMed/OpenMed-NER-GenomicDetect-BioMed-335M/model.ort
3
+ ai.onnx;7;And
4
+ ai.onnx;11;Range{"inputs": {"0": ["int64_t"]}}
5
+ ai.onnx;13;Cast{"inputs": {"0": ["int64_t"]}, "outputs": {"0": ["bool"]}},Concat{"inputs": {"0": ["int64_t"]}},Equal{"inputs": {"0": ["int64_t"]}},Expand{"inputs": {"0": ["bool"]}},Flatten,Gather{"inputs": {"0": ["bool", "float", "int64_t"], "1": ["int64_t"]}},IsNaN{"inputs": {"0": ["float"]}},MatMul{"inputs": {"0": ["float"]}},Slice{"inputs": {"0": ["int64_t"], "1": ["int64_t"]}},Softmax{"inputs": {"0": ["float"]}},Transpose{"inputs": {"0": ["float"]}},Unsqueeze
6
+ ai.onnx;14;Add{"inputs": {"0": ["float", "int64_t"]}},Mul{"inputs": {"0": ["int64_t"]}},Reshape
7
+ ai.onnx;15;Shape
8
+ ai.onnx;16;GreaterOrEqual,Where{"inputs": {"1": ["float", "int64_t"]}}
9
+ ai.onnx;17;LayerNormalization
10
+ com.microsoft;1;BiasGelu,FusedMatMul
model_fp16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93fb0af24b365171ceca1d982c828d138f129bbd4b04e253192ebbf8032f7cb0
3
+ size 668856285
model_int8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b10a2d0947307d22059397c2bfb99f20f51330b72344dfdb0ed27919c6e4ae7
3
+ size 431283161
openmed-onnx.json ADDED
@@ -0,0 +1,517 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "openmed-onnx",
3
+ "format_version": 1,
4
+ "formats": [
5
+ "onnx-android",
6
+ "onnx-int8",
7
+ "ort-android"
8
+ ],
9
+ "task": "token-classification",
10
+ "family": "bert",
11
+ "source_model_id": "OpenMed/OpenMed-NER-GenomicDetect-BioMed-335M",
12
+ "config_path": "config.json",
13
+ "label_map_path": "id2label.json",
14
+ "artifacts": [
15
+ {
16
+ "format": "onnx-android",
17
+ "path": "model.onnx",
18
+ "precision": "float32",
19
+ "metadata": {
20
+ "profile": "android",
21
+ "opset": 18,
22
+ "execution_providers": [
23
+ "NNAPI",
24
+ "XNNPACK"
25
+ ],
26
+ "inputs": [
27
+ {
28
+ "name": "input_ids",
29
+ "dtype": "int64",
30
+ "axes": [
31
+ "batch",
32
+ "sequence"
33
+ ],
34
+ "shape": [
35
+ {
36
+ "kind": "dynamic",
37
+ "name": "batch"
38
+ },
39
+ {
40
+ "kind": "dynamic",
41
+ "name": "sequence"
42
+ }
43
+ ]
44
+ },
45
+ {
46
+ "name": "attention_mask",
47
+ "dtype": "int64",
48
+ "axes": [
49
+ "batch",
50
+ "sequence"
51
+ ],
52
+ "shape": [
53
+ {
54
+ "kind": "dynamic",
55
+ "name": "batch"
56
+ },
57
+ {
58
+ "kind": "dynamic",
59
+ "name": "sequence"
60
+ }
61
+ ]
62
+ },
63
+ {
64
+ "name": "token_type_ids",
65
+ "dtype": "int64",
66
+ "axes": [
67
+ "batch",
68
+ "sequence"
69
+ ],
70
+ "shape": [
71
+ {
72
+ "kind": "dynamic",
73
+ "name": "batch"
74
+ },
75
+ {
76
+ "kind": "dynamic",
77
+ "name": "sequence"
78
+ }
79
+ ]
80
+ }
81
+ ],
82
+ "outputs": [
83
+ {
84
+ "name": "logits",
85
+ "dtype": "float32",
86
+ "axes": [
87
+ "batch",
88
+ "sequence",
89
+ "labels"
90
+ ],
91
+ "shape": [
92
+ {
93
+ "kind": "dynamic",
94
+ "name": "batch"
95
+ },
96
+ {
97
+ "kind": "dynamic",
98
+ "name": "sequence"
99
+ },
100
+ {
101
+ "kind": "static",
102
+ "value": 3
103
+ }
104
+ ]
105
+ }
106
+ ],
107
+ "operators": [
108
+ "Shape",
109
+ "Constant",
110
+ "Gather",
111
+ "Unsqueeze",
112
+ "Slice",
113
+ "Add",
114
+ "LayerNormalization",
115
+ "Cast",
116
+ "Range",
117
+ "ConstantOfShape",
118
+ "GreaterOrEqual",
119
+ "And",
120
+ "Flatten",
121
+ "Mul",
122
+ "Reshape",
123
+ "Concat",
124
+ "Equal",
125
+ "Where",
126
+ "Expand",
127
+ "MatMul",
128
+ "Transpose",
129
+ "Softmax",
130
+ "IsNaN",
131
+ "Div",
132
+ "Erf"
133
+ ],
134
+ "unsupported_ops": [
135
+ "GreaterOrEqual",
136
+ "Flatten",
137
+ "IsNaN"
138
+ ],
139
+ "warnings": [
140
+ "GreaterOrEqual is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
141
+ "Flatten is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
142
+ "IsNaN is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set"
143
+ ]
144
+ }
145
+ },
146
+ {
147
+ "format": "onnx-android",
148
+ "path": "model_fp16.onnx",
149
+ "precision": "float16",
150
+ "metadata": {
151
+ "profile": "android",
152
+ "opset": 18,
153
+ "execution_providers": [
154
+ "NNAPI",
155
+ "XNNPACK"
156
+ ],
157
+ "inputs": [
158
+ {
159
+ "name": "input_ids",
160
+ "dtype": "int64",
161
+ "axes": [
162
+ "batch",
163
+ "sequence"
164
+ ],
165
+ "shape": [
166
+ {
167
+ "kind": "dynamic",
168
+ "name": "batch"
169
+ },
170
+ {
171
+ "kind": "dynamic",
172
+ "name": "sequence"
173
+ }
174
+ ]
175
+ },
176
+ {
177
+ "name": "attention_mask",
178
+ "dtype": "int64",
179
+ "axes": [
180
+ "batch",
181
+ "sequence"
182
+ ],
183
+ "shape": [
184
+ {
185
+ "kind": "dynamic",
186
+ "name": "batch"
187
+ },
188
+ {
189
+ "kind": "dynamic",
190
+ "name": "sequence"
191
+ }
192
+ ]
193
+ },
194
+ {
195
+ "name": "token_type_ids",
196
+ "dtype": "int64",
197
+ "axes": [
198
+ "batch",
199
+ "sequence"
200
+ ],
201
+ "shape": [
202
+ {
203
+ "kind": "dynamic",
204
+ "name": "batch"
205
+ },
206
+ {
207
+ "kind": "dynamic",
208
+ "name": "sequence"
209
+ }
210
+ ]
211
+ }
212
+ ],
213
+ "outputs": [
214
+ {
215
+ "name": "logits",
216
+ "dtype": "float32",
217
+ "axes": [
218
+ "batch",
219
+ "sequence",
220
+ "labels"
221
+ ],
222
+ "shape": [
223
+ {
224
+ "kind": "dynamic",
225
+ "name": "batch"
226
+ },
227
+ {
228
+ "kind": "dynamic",
229
+ "name": "sequence"
230
+ },
231
+ {
232
+ "kind": "static",
233
+ "value": 3
234
+ }
235
+ ]
236
+ }
237
+ ],
238
+ "operators": [
239
+ "Shape",
240
+ "Constant",
241
+ "Gather",
242
+ "Unsqueeze",
243
+ "Slice",
244
+ "Add",
245
+ "LayerNormalization",
246
+ "Cast",
247
+ "Range",
248
+ "ConstantOfShape",
249
+ "GreaterOrEqual",
250
+ "And",
251
+ "Flatten",
252
+ "Mul",
253
+ "Reshape",
254
+ "Concat",
255
+ "Equal",
256
+ "Where",
257
+ "Expand",
258
+ "MatMul",
259
+ "Transpose",
260
+ "Softmax",
261
+ "IsNaN",
262
+ "Div",
263
+ "Erf"
264
+ ],
265
+ "unsupported_ops": [
266
+ "GreaterOrEqual",
267
+ "Flatten",
268
+ "IsNaN"
269
+ ],
270
+ "warnings": [
271
+ "GreaterOrEqual is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
272
+ "Flatten is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
273
+ "IsNaN is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set"
274
+ ]
275
+ }
276
+ },
277
+ {
278
+ "format": "onnx-int8",
279
+ "path": "model_int8.onnx",
280
+ "precision": "int8",
281
+ "metadata": {
282
+ "format": "onnx-int8",
283
+ "quantization": {
284
+ "scheme": "dynamic",
285
+ "weight_type": "qint8",
286
+ "op_types": [
287
+ "MatMul",
288
+ "Gemm"
289
+ ],
290
+ "calibration": "none",
291
+ "target": "arm64-cpu",
292
+ "output": "model_int8.onnx"
293
+ },
294
+ "profile": "android",
295
+ "opset": 18,
296
+ "execution_providers": [
297
+ "NNAPI",
298
+ "XNNPACK"
299
+ ],
300
+ "inputs": [
301
+ {
302
+ "name": "input_ids",
303
+ "dtype": "int64",
304
+ "axes": [
305
+ "batch",
306
+ "sequence"
307
+ ],
308
+ "shape": [
309
+ {
310
+ "kind": "dynamic",
311
+ "name": "batch"
312
+ },
313
+ {
314
+ "kind": "dynamic",
315
+ "name": "sequence"
316
+ }
317
+ ]
318
+ },
319
+ {
320
+ "name": "attention_mask",
321
+ "dtype": "int64",
322
+ "axes": [
323
+ "batch",
324
+ "sequence"
325
+ ],
326
+ "shape": [
327
+ {
328
+ "kind": "dynamic",
329
+ "name": "batch"
330
+ },
331
+ {
332
+ "kind": "dynamic",
333
+ "name": "sequence"
334
+ }
335
+ ]
336
+ },
337
+ {
338
+ "name": "token_type_ids",
339
+ "dtype": "int64",
340
+ "axes": [
341
+ "batch",
342
+ "sequence"
343
+ ],
344
+ "shape": [
345
+ {
346
+ "kind": "dynamic",
347
+ "name": "batch"
348
+ },
349
+ {
350
+ "kind": "dynamic",
351
+ "name": "sequence"
352
+ }
353
+ ]
354
+ }
355
+ ],
356
+ "outputs": [
357
+ {
358
+ "name": "logits",
359
+ "dtype": "float32",
360
+ "axes": [
361
+ "batch",
362
+ "sequence",
363
+ "labels"
364
+ ],
365
+ "shape": [
366
+ {
367
+ "kind": "dynamic",
368
+ "name": "batch"
369
+ },
370
+ {
371
+ "kind": "dynamic",
372
+ "name": "sequence"
373
+ },
374
+ {
375
+ "kind": "static",
376
+ "value": 3
377
+ }
378
+ ]
379
+ }
380
+ ],
381
+ "operators": [
382
+ "Constant",
383
+ "Cast",
384
+ "Shape",
385
+ "Gather",
386
+ "ConstantOfShape",
387
+ "Flatten",
388
+ "Add",
389
+ "Unsqueeze",
390
+ "Slice",
391
+ "Range",
392
+ "LayerNormalization",
393
+ "DynamicQuantizeLinear",
394
+ "Mul",
395
+ "MatMulInteger",
396
+ "Concat",
397
+ "Reshape",
398
+ "Transpose",
399
+ "GreaterOrEqual",
400
+ "Equal",
401
+ "MatMul",
402
+ "And",
403
+ "Where",
404
+ "Expand",
405
+ "Softmax",
406
+ "IsNaN",
407
+ "Div",
408
+ "Erf"
409
+ ],
410
+ "unsupported_ops": [
411
+ "Flatten",
412
+ "DynamicQuantizeLinear",
413
+ "MatMulInteger",
414
+ "GreaterOrEqual",
415
+ "IsNaN"
416
+ ],
417
+ "warnings": [
418
+ "Flatten is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
419
+ "DynamicQuantizeLinear is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
420
+ "MatMulInteger is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
421
+ "GreaterOrEqual is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set",
422
+ "IsNaN is not in the Android ONNX Runtime Mobile NNAPI/XNNPACK safe set"
423
+ ]
424
+ }
425
+ },
426
+ {
427
+ "format": "ort-android",
428
+ "path": "model.ort",
429
+ "precision": "float32",
430
+ "metadata": {
431
+ "format": "ort-android",
432
+ "profile": "android",
433
+ "source_onnx_path": "model.onnx",
434
+ "optimization_style": "Fixed",
435
+ "target_platform": "arm",
436
+ "op_config_type": "required_operators_and_types",
437
+ "enable_type_reduction": true,
438
+ "ort_path": "model.ort",
439
+ "op_config_path": "model.required_operators_and_types.config",
440
+ "operators": [
441
+ "Shape",
442
+ "Constant",
443
+ "Gather",
444
+ "Unsqueeze",
445
+ "Slice",
446
+ "Add",
447
+ "LayerNormalization",
448
+ "Cast",
449
+ "Range",
450
+ "ConstantOfShape",
451
+ "GreaterOrEqual",
452
+ "And",
453
+ "Flatten",
454
+ "Mul",
455
+ "Reshape",
456
+ "Concat",
457
+ "Equal",
458
+ "Where",
459
+ "Expand",
460
+ "MatMul",
461
+ "Transpose",
462
+ "Softmax",
463
+ "IsNaN",
464
+ "Div",
465
+ "Erf"
466
+ ],
467
+ "opset": 18
468
+ }
469
+ }
470
+ ],
471
+ "minimum_opset": 18,
472
+ "max_sequence_length": 512,
473
+ "dynamic_shapes": {
474
+ "batch_axis": "dynamic",
475
+ "sequence_axis": "dynamic",
476
+ "shape_buckets": {
477
+ "strategy": "next_configured_bucket_or_exact",
478
+ "axes": {
479
+ "input": [
480
+ "batch",
481
+ "sequence"
482
+ ],
483
+ "output": [
484
+ "batch",
485
+ "sequence"
486
+ ]
487
+ },
488
+ "buckets": [
489
+ 8,
490
+ 16,
491
+ 32,
492
+ 64,
493
+ 128,
494
+ 256,
495
+ 512,
496
+ 1024,
497
+ 2048
498
+ ],
499
+ "min_length": 8,
500
+ "max_length": 2048,
501
+ "overflow": "exact_length"
502
+ }
503
+ },
504
+ "optimization": {
505
+ "enabled": false,
506
+ "minimum_opset": 18
507
+ },
508
+ "validation": {},
509
+ "operator_fallbacks": [],
510
+ "tokenizer": {
511
+ "path": ".",
512
+ "files": [
513
+ "tokenizer.json",
514
+ "tokenizer_config.json"
515
+ ]
516
+ }
517
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "clean_up_tokenization_spaces": true,
4
+ "cls_token": "[CLS]",
5
+ "do_basic_tokenize": true,
6
+ "do_lower_case": true,
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "mask_token": "[MASK]",
10
+ "max_length": 512,
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "never_split": null,
13
+ "pad_to_multiple_of": null,
14
+ "pad_token": "[PAD]",
15
+ "pad_token_type_id": 0,
16
+ "padding_side": "right",
17
+ "sep_token": "[SEP]",
18
+ "stride": 0,
19
+ "strip_accents": null,
20
+ "tokenize_chinese_chars": true,
21
+ "tokenizer_class": "BertTokenizer",
22
+ "truncation_side": "right",
23
+ "truncation_strategy": "longest_first",
24
+ "unk_token": "[UNK]"
25
+ }