Training done
Browse files- added_tokens.json +4 -0
- tokenizer.json +36 -0
- tokenizer_config.json +32 -0
added_tokens.json
CHANGED
|
@@ -17,8 +17,10 @@
|
|
| 17 |
"</s_item_quantity_shipped>": 57538,
|
| 18 |
"</s_item_quantity_short>": 57569,
|
| 19 |
"</s_item_total_price>": 57544,
|
|
|
|
| 20 |
"</s_item_unit>": 57563,
|
| 21 |
"</s_item_unit_price>": 57542,
|
|
|
|
| 22 |
"</s_item_uom>": 57534,
|
| 23 |
"</s_item_weight>": 57536,
|
| 24 |
"</s_items>": 57526,
|
|
@@ -43,8 +45,10 @@
|
|
| 43 |
"<s_item_quantity_shipped>": 57537,
|
| 44 |
"<s_item_quantity_short>": 57568,
|
| 45 |
"<s_item_total_price>": 57543,
|
|
|
|
| 46 |
"<s_item_unit>": 57562,
|
| 47 |
"<s_item_unit_price>": 57541,
|
|
|
|
| 48 |
"<s_item_uom>": 57533,
|
| 49 |
"<s_item_weight>": 57535,
|
| 50 |
"<s_items>": 57525,
|
|
|
|
| 17 |
"</s_item_quantity_shipped>": 57538,
|
| 18 |
"</s_item_quantity_short>": 57569,
|
| 19 |
"</s_item_total_price>": 57544,
|
| 20 |
+
"</s_item_total_quantity>": 57577,
|
| 21 |
"</s_item_unit>": 57563,
|
| 22 |
"</s_item_unit_price>": 57542,
|
| 23 |
+
"</s_item_unit_quantity>": 57575,
|
| 24 |
"</s_item_uom>": 57534,
|
| 25 |
"</s_item_weight>": 57536,
|
| 26 |
"</s_items>": 57526,
|
|
|
|
| 45 |
"<s_item_quantity_shipped>": 57537,
|
| 46 |
"<s_item_quantity_short>": 57568,
|
| 47 |
"<s_item_total_price>": 57543,
|
| 48 |
+
"<s_item_total_quantity>": 57576,
|
| 49 |
"<s_item_unit>": 57562,
|
| 50 |
"<s_item_unit_price>": 57541,
|
| 51 |
+
"<s_item_unit_quantity>": 57574,
|
| 52 |
"<s_item_uom>": 57533,
|
| 53 |
"<s_item_weight>": 57535,
|
| 54 |
"<s_items>": 57525,
|
tokenizer.json
CHANGED
|
@@ -515,6 +515,42 @@
|
|
| 515 |
"rstrip": false,
|
| 516 |
"normalized": true,
|
| 517 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 518 |
}
|
| 519 |
],
|
| 520 |
"normalizer": {
|
|
|
|
| 515 |
"rstrip": false,
|
| 516 |
"normalized": true,
|
| 517 |
"special": false
|
| 518 |
+
},
|
| 519 |
+
{
|
| 520 |
+
"id": 57574,
|
| 521 |
+
"content": "<s_item_unit_quantity>",
|
| 522 |
+
"single_word": false,
|
| 523 |
+
"lstrip": false,
|
| 524 |
+
"rstrip": false,
|
| 525 |
+
"normalized": true,
|
| 526 |
+
"special": false
|
| 527 |
+
},
|
| 528 |
+
{
|
| 529 |
+
"id": 57575,
|
| 530 |
+
"content": "</s_item_unit_quantity>",
|
| 531 |
+
"single_word": false,
|
| 532 |
+
"lstrip": false,
|
| 533 |
+
"rstrip": false,
|
| 534 |
+
"normalized": true,
|
| 535 |
+
"special": false
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"id": 57576,
|
| 539 |
+
"content": "<s_item_total_quantity>",
|
| 540 |
+
"single_word": false,
|
| 541 |
+
"lstrip": false,
|
| 542 |
+
"rstrip": false,
|
| 543 |
+
"normalized": true,
|
| 544 |
+
"special": false
|
| 545 |
+
},
|
| 546 |
+
{
|
| 547 |
+
"id": 57577,
|
| 548 |
+
"content": "</s_item_total_quantity>",
|
| 549 |
+
"single_word": false,
|
| 550 |
+
"lstrip": false,
|
| 551 |
+
"rstrip": false,
|
| 552 |
+
"normalized": true,
|
| 553 |
+
"special": false
|
| 554 |
}
|
| 555 |
],
|
| 556 |
"normalizer": {
|
tokenizer_config.json
CHANGED
|
@@ -455,6 +455,38 @@
|
|
| 455 |
"rstrip": false,
|
| 456 |
"single_word": false,
|
| 457 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
}
|
| 459 |
},
|
| 460 |
"additional_special_tokens": [
|
|
|
|
| 455 |
"rstrip": false,
|
| 456 |
"single_word": false,
|
| 457 |
"special": false
|
| 458 |
+
},
|
| 459 |
+
"57574": {
|
| 460 |
+
"content": "<s_item_unit_quantity>",
|
| 461 |
+
"lstrip": false,
|
| 462 |
+
"normalized": true,
|
| 463 |
+
"rstrip": false,
|
| 464 |
+
"single_word": false,
|
| 465 |
+
"special": false
|
| 466 |
+
},
|
| 467 |
+
"57575": {
|
| 468 |
+
"content": "</s_item_unit_quantity>",
|
| 469 |
+
"lstrip": false,
|
| 470 |
+
"normalized": true,
|
| 471 |
+
"rstrip": false,
|
| 472 |
+
"single_word": false,
|
| 473 |
+
"special": false
|
| 474 |
+
},
|
| 475 |
+
"57576": {
|
| 476 |
+
"content": "<s_item_total_quantity>",
|
| 477 |
+
"lstrip": false,
|
| 478 |
+
"normalized": true,
|
| 479 |
+
"rstrip": false,
|
| 480 |
+
"single_word": false,
|
| 481 |
+
"special": false
|
| 482 |
+
},
|
| 483 |
+
"57577": {
|
| 484 |
+
"content": "</s_item_total_quantity>",
|
| 485 |
+
"lstrip": false,
|
| 486 |
+
"normalized": true,
|
| 487 |
+
"rstrip": false,
|
| 488 |
+
"single_word": false,
|
| 489 |
+
"special": false
|
| 490 |
}
|
| 491 |
},
|
| 492 |
"additional_special_tokens": [
|