Training done
Browse files- added_tokens.json +4 -0
- tokenizer.json +52 -2
- tokenizer_config.json +32 -0
added_tokens.json
CHANGED
|
@@ -62,7 +62,9 @@
|
|
| 62 |
"</s_shipper_email>": 57544,
|
| 63 |
"</s_shipper_name>": 57538,
|
| 64 |
"</s_shipper_no>": 57702,
|
|
|
|
| 65 |
"</s_shipper_phone_no>": 57542,
|
|
|
|
| 66 |
"</s_shipper_pin>": 57546,
|
| 67 |
"</s_shipping_address>": 57649,
|
| 68 |
"</s_shipping_date>": 57586,
|
|
@@ -153,7 +155,9 @@
|
|
| 153 |
"<s_shipper_email>": 57543,
|
| 154 |
"<s_shipper_name>": 57537,
|
| 155 |
"<s_shipper_no>": 57701,
|
|
|
|
| 156 |
"<s_shipper_phone_no>": 57541,
|
|
|
|
| 157 |
"<s_shipper_pin>": 57545,
|
| 158 |
"<s_shipping_address>": 57648,
|
| 159 |
"<s_shipping_date>": 57585,
|
|
|
|
| 62 |
"</s_shipper_email>": 57544,
|
| 63 |
"</s_shipper_name>": 57538,
|
| 64 |
"</s_shipper_no>": 57702,
|
| 65 |
+
"</s_shipper_number>": 57704,
|
| 66 |
"</s_shipper_phone_no>": 57542,
|
| 67 |
+
"</s_shipper_phone_number>": 57706,
|
| 68 |
"</s_shipper_pin>": 57546,
|
| 69 |
"</s_shipping_address>": 57649,
|
| 70 |
"</s_shipping_date>": 57586,
|
|
|
|
| 155 |
"<s_shipper_email>": 57543,
|
| 156 |
"<s_shipper_name>": 57537,
|
| 157 |
"<s_shipper_no>": 57701,
|
| 158 |
+
"<s_shipper_number>": 57703,
|
| 159 |
"<s_shipper_phone_no>": 57541,
|
| 160 |
+
"<s_shipper_phone_number>": 57705,
|
| 161 |
"<s_shipper_pin>": 57545,
|
| 162 |
"<s_shipping_address>": 57648,
|
| 163 |
"<s_shipping_date>": 57585,
|
tokenizer.json
CHANGED
|
@@ -1,7 +1,21 @@
|
|
| 1 |
{
|
| 2 |
"version": "1.0",
|
| 3 |
-
"truncation":
|
| 4 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
"added_tokens": [
|
| 6 |
{
|
| 7 |
"id": 0,
|
|
@@ -1676,6 +1690,42 @@
|
|
| 1676 |
"rstrip": false,
|
| 1677 |
"normalized": true,
|
| 1678 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1679 |
}
|
| 1680 |
],
|
| 1681 |
"normalizer": {
|
|
|
|
| 1 |
{
|
| 2 |
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 768,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": {
|
| 10 |
+
"strategy": {
|
| 11 |
+
"Fixed": 768
|
| 12 |
+
},
|
| 13 |
+
"direction": "Right",
|
| 14 |
+
"pad_to_multiple_of": null,
|
| 15 |
+
"pad_id": 1,
|
| 16 |
+
"pad_type_id": 0,
|
| 17 |
+
"pad_token": "<pad>"
|
| 18 |
+
},
|
| 19 |
"added_tokens": [
|
| 20 |
{
|
| 21 |
"id": 0,
|
|
|
|
| 1690 |
"rstrip": false,
|
| 1691 |
"normalized": true,
|
| 1692 |
"special": false
|
| 1693 |
+
},
|
| 1694 |
+
{
|
| 1695 |
+
"id": 57703,
|
| 1696 |
+
"content": "<s_shipper_number>",
|
| 1697 |
+
"single_word": false,
|
| 1698 |
+
"lstrip": false,
|
| 1699 |
+
"rstrip": false,
|
| 1700 |
+
"normalized": true,
|
| 1701 |
+
"special": false
|
| 1702 |
+
},
|
| 1703 |
+
{
|
| 1704 |
+
"id": 57704,
|
| 1705 |
+
"content": "</s_shipper_number>",
|
| 1706 |
+
"single_word": false,
|
| 1707 |
+
"lstrip": false,
|
| 1708 |
+
"rstrip": false,
|
| 1709 |
+
"normalized": true,
|
| 1710 |
+
"special": false
|
| 1711 |
+
},
|
| 1712 |
+
{
|
| 1713 |
+
"id": 57705,
|
| 1714 |
+
"content": "<s_shipper_phone_number>",
|
| 1715 |
+
"single_word": false,
|
| 1716 |
+
"lstrip": false,
|
| 1717 |
+
"rstrip": false,
|
| 1718 |
+
"normalized": true,
|
| 1719 |
+
"special": false
|
| 1720 |
+
},
|
| 1721 |
+
{
|
| 1722 |
+
"id": 57706,
|
| 1723 |
+
"content": "</s_shipper_phone_number>",
|
| 1724 |
+
"single_word": false,
|
| 1725 |
+
"lstrip": false,
|
| 1726 |
+
"rstrip": false,
|
| 1727 |
+
"normalized": true,
|
| 1728 |
+
"special": false
|
| 1729 |
}
|
| 1730 |
],
|
| 1731 |
"normalizer": {
|
tokenizer_config.json
CHANGED
|
@@ -1487,6 +1487,38 @@
|
|
| 1487 |
"rstrip": false,
|
| 1488 |
"single_word": false,
|
| 1489 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1490 |
}
|
| 1491 |
},
|
| 1492 |
"additional_special_tokens": [
|
|
|
|
| 1487 |
"rstrip": false,
|
| 1488 |
"single_word": false,
|
| 1489 |
"special": false
|
| 1490 |
+
},
|
| 1491 |
+
"57703": {
|
| 1492 |
+
"content": "<s_shipper_number>",
|
| 1493 |
+
"lstrip": false,
|
| 1494 |
+
"normalized": true,
|
| 1495 |
+
"rstrip": false,
|
| 1496 |
+
"single_word": false,
|
| 1497 |
+
"special": false
|
| 1498 |
+
},
|
| 1499 |
+
"57704": {
|
| 1500 |
+
"content": "</s_shipper_number>",
|
| 1501 |
+
"lstrip": false,
|
| 1502 |
+
"normalized": true,
|
| 1503 |
+
"rstrip": false,
|
| 1504 |
+
"single_word": false,
|
| 1505 |
+
"special": false
|
| 1506 |
+
},
|
| 1507 |
+
"57705": {
|
| 1508 |
+
"content": "<s_shipper_phone_number>",
|
| 1509 |
+
"lstrip": false,
|
| 1510 |
+
"normalized": true,
|
| 1511 |
+
"rstrip": false,
|
| 1512 |
+
"single_word": false,
|
| 1513 |
+
"special": false
|
| 1514 |
+
},
|
| 1515 |
+
"57706": {
|
| 1516 |
+
"content": "</s_shipper_phone_number>",
|
| 1517 |
+
"lstrip": false,
|
| 1518 |
+
"normalized": true,
|
| 1519 |
+
"rstrip": false,
|
| 1520 |
+
"single_word": false,
|
| 1521 |
+
"special": false
|
| 1522 |
}
|
| 1523 |
},
|
| 1524 |
"additional_special_tokens": [
|