Training done
Browse files- added_tokens.json +14 -0
- tokenizer.json +126 -0
- tokenizer_config.json +112 -0
added_tokens.json
CHANGED
|
@@ -50,6 +50,11 @@
|
|
| 50 |
"</s_receipt_date>": 57609,
|
| 51 |
"</s_receipt_voucher_no>": 57607,
|
| 52 |
"</s_reference_no>": 57578,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
"</s_section>": 57627,
|
| 54 |
"</s_ship_to>": 57647,
|
| 55 |
"</s_shipper>": 57536,
|
|
@@ -67,8 +72,10 @@
|
|
| 67 |
"</s_supplier_address>": 57635,
|
| 68 |
"</s_supplier_code>": 57601,
|
| 69 |
"</s_supplier_email>": 57617,
|
|
|
|
| 70 |
"</s_supplier_name>": 57603,
|
| 71 |
"</s_supplier_number>": 57633,
|
|
|
|
| 72 |
"</s_supplier_pin>": 57637,
|
| 73 |
"</s_supplier_tel>": 57639,
|
| 74 |
"</s_total_weight>": 57554,
|
|
@@ -133,6 +140,11 @@
|
|
| 133 |
"<s_receipt_date>": 57608,
|
| 134 |
"<s_receipt_voucher_no>": 57606,
|
| 135 |
"<s_reference_no>": 57577,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
"<s_section>": 57626,
|
| 137 |
"<s_ship_to>": 57646,
|
| 138 |
"<s_shipper>": 57535,
|
|
@@ -150,8 +162,10 @@
|
|
| 150 |
"<s_supplier_address>": 57634,
|
| 151 |
"<s_supplier_code>": 57600,
|
| 152 |
"<s_supplier_email>": 57616,
|
|
|
|
| 153 |
"<s_supplier_name>": 57602,
|
| 154 |
"<s_supplier_number>": 57632,
|
|
|
|
| 155 |
"<s_supplier_pin>": 57636,
|
| 156 |
"<s_supplier_tel>": 57638,
|
| 157 |
"<s_synthdog>": 57524,
|
|
|
|
| 50 |
"</s_receipt_date>": 57609,
|
| 51 |
"</s_receipt_voucher_no>": 57607,
|
| 52 |
"</s_reference_no>": 57578,
|
| 53 |
+
"</s_retailer>": 57688,
|
| 54 |
+
"</s_retailer_address>": 57692,
|
| 55 |
+
"</s_retailer_name>": 57690,
|
| 56 |
+
"</s_retailer_phone_number>": 57696,
|
| 57 |
+
"</s_retailer_pin>": 57694,
|
| 58 |
"</s_section>": 57627,
|
| 59 |
"</s_ship_to>": 57647,
|
| 60 |
"</s_shipper>": 57536,
|
|
|
|
| 72 |
"</s_supplier_address>": 57635,
|
| 73 |
"</s_supplier_code>": 57601,
|
| 74 |
"</s_supplier_email>": 57617,
|
| 75 |
+
"</s_supplier_id>": 57698,
|
| 76 |
"</s_supplier_name>": 57603,
|
| 77 |
"</s_supplier_number>": 57633,
|
| 78 |
+
"</s_supplier_phone_number>": 57700,
|
| 79 |
"</s_supplier_pin>": 57637,
|
| 80 |
"</s_supplier_tel>": 57639,
|
| 81 |
"</s_total_weight>": 57554,
|
|
|
|
| 140 |
"<s_receipt_date>": 57608,
|
| 141 |
"<s_receipt_voucher_no>": 57606,
|
| 142 |
"<s_reference_no>": 57577,
|
| 143 |
+
"<s_retailer>": 57687,
|
| 144 |
+
"<s_retailer_address>": 57691,
|
| 145 |
+
"<s_retailer_name>": 57689,
|
| 146 |
+
"<s_retailer_phone_number>": 57695,
|
| 147 |
+
"<s_retailer_pin>": 57693,
|
| 148 |
"<s_section>": 57626,
|
| 149 |
"<s_ship_to>": 57646,
|
| 150 |
"<s_shipper>": 57535,
|
|
|
|
| 162 |
"<s_supplier_address>": 57634,
|
| 163 |
"<s_supplier_code>": 57600,
|
| 164 |
"<s_supplier_email>": 57616,
|
| 165 |
+
"<s_supplier_id>": 57697,
|
| 166 |
"<s_supplier_name>": 57602,
|
| 167 |
"<s_supplier_number>": 57632,
|
| 168 |
+
"<s_supplier_phone_number>": 57699,
|
| 169 |
"<s_supplier_pin>": 57636,
|
| 170 |
"<s_supplier_tel>": 57638,
|
| 171 |
"<s_synthdog>": 57524,
|
tokenizer.json
CHANGED
|
@@ -1532,6 +1532,132 @@
|
|
| 1532 |
"rstrip": false,
|
| 1533 |
"normalized": true,
|
| 1534 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1535 |
}
|
| 1536 |
],
|
| 1537 |
"normalizer": {
|
|
|
|
| 1532 |
"rstrip": false,
|
| 1533 |
"normalized": true,
|
| 1534 |
"special": false
|
| 1535 |
+
},
|
| 1536 |
+
{
|
| 1537 |
+
"id": 57687,
|
| 1538 |
+
"content": "<s_retailer>",
|
| 1539 |
+
"single_word": false,
|
| 1540 |
+
"lstrip": false,
|
| 1541 |
+
"rstrip": false,
|
| 1542 |
+
"normalized": true,
|
| 1543 |
+
"special": false
|
| 1544 |
+
},
|
| 1545 |
+
{
|
| 1546 |
+
"id": 57688,
|
| 1547 |
+
"content": "</s_retailer>",
|
| 1548 |
+
"single_word": false,
|
| 1549 |
+
"lstrip": false,
|
| 1550 |
+
"rstrip": false,
|
| 1551 |
+
"normalized": true,
|
| 1552 |
+
"special": false
|
| 1553 |
+
},
|
| 1554 |
+
{
|
| 1555 |
+
"id": 57689,
|
| 1556 |
+
"content": "<s_retailer_name>",
|
| 1557 |
+
"single_word": false,
|
| 1558 |
+
"lstrip": false,
|
| 1559 |
+
"rstrip": false,
|
| 1560 |
+
"normalized": true,
|
| 1561 |
+
"special": false
|
| 1562 |
+
},
|
| 1563 |
+
{
|
| 1564 |
+
"id": 57690,
|
| 1565 |
+
"content": "</s_retailer_name>",
|
| 1566 |
+
"single_word": false,
|
| 1567 |
+
"lstrip": false,
|
| 1568 |
+
"rstrip": false,
|
| 1569 |
+
"normalized": true,
|
| 1570 |
+
"special": false
|
| 1571 |
+
},
|
| 1572 |
+
{
|
| 1573 |
+
"id": 57691,
|
| 1574 |
+
"content": "<s_retailer_address>",
|
| 1575 |
+
"single_word": false,
|
| 1576 |
+
"lstrip": false,
|
| 1577 |
+
"rstrip": false,
|
| 1578 |
+
"normalized": true,
|
| 1579 |
+
"special": false
|
| 1580 |
+
},
|
| 1581 |
+
{
|
| 1582 |
+
"id": 57692,
|
| 1583 |
+
"content": "</s_retailer_address>",
|
| 1584 |
+
"single_word": false,
|
| 1585 |
+
"lstrip": false,
|
| 1586 |
+
"rstrip": false,
|
| 1587 |
+
"normalized": true,
|
| 1588 |
+
"special": false
|
| 1589 |
+
},
|
| 1590 |
+
{
|
| 1591 |
+
"id": 57693,
|
| 1592 |
+
"content": "<s_retailer_pin>",
|
| 1593 |
+
"single_word": false,
|
| 1594 |
+
"lstrip": false,
|
| 1595 |
+
"rstrip": false,
|
| 1596 |
+
"normalized": true,
|
| 1597 |
+
"special": false
|
| 1598 |
+
},
|
| 1599 |
+
{
|
| 1600 |
+
"id": 57694,
|
| 1601 |
+
"content": "</s_retailer_pin>",
|
| 1602 |
+
"single_word": false,
|
| 1603 |
+
"lstrip": false,
|
| 1604 |
+
"rstrip": false,
|
| 1605 |
+
"normalized": true,
|
| 1606 |
+
"special": false
|
| 1607 |
+
},
|
| 1608 |
+
{
|
| 1609 |
+
"id": 57695,
|
| 1610 |
+
"content": "<s_retailer_phone_number>",
|
| 1611 |
+
"single_word": false,
|
| 1612 |
+
"lstrip": false,
|
| 1613 |
+
"rstrip": false,
|
| 1614 |
+
"normalized": true,
|
| 1615 |
+
"special": false
|
| 1616 |
+
},
|
| 1617 |
+
{
|
| 1618 |
+
"id": 57696,
|
| 1619 |
+
"content": "</s_retailer_phone_number>",
|
| 1620 |
+
"single_word": false,
|
| 1621 |
+
"lstrip": false,
|
| 1622 |
+
"rstrip": false,
|
| 1623 |
+
"normalized": true,
|
| 1624 |
+
"special": false
|
| 1625 |
+
},
|
| 1626 |
+
{
|
| 1627 |
+
"id": 57697,
|
| 1628 |
+
"content": "<s_supplier_id>",
|
| 1629 |
+
"single_word": false,
|
| 1630 |
+
"lstrip": false,
|
| 1631 |
+
"rstrip": false,
|
| 1632 |
+
"normalized": true,
|
| 1633 |
+
"special": false
|
| 1634 |
+
},
|
| 1635 |
+
{
|
| 1636 |
+
"id": 57698,
|
| 1637 |
+
"content": "</s_supplier_id>",
|
| 1638 |
+
"single_word": false,
|
| 1639 |
+
"lstrip": false,
|
| 1640 |
+
"rstrip": false,
|
| 1641 |
+
"normalized": true,
|
| 1642 |
+
"special": false
|
| 1643 |
+
},
|
| 1644 |
+
{
|
| 1645 |
+
"id": 57699,
|
| 1646 |
+
"content": "<s_supplier_phone_number>",
|
| 1647 |
+
"single_word": false,
|
| 1648 |
+
"lstrip": false,
|
| 1649 |
+
"rstrip": false,
|
| 1650 |
+
"normalized": true,
|
| 1651 |
+
"special": false
|
| 1652 |
+
},
|
| 1653 |
+
{
|
| 1654 |
+
"id": 57700,
|
| 1655 |
+
"content": "</s_supplier_phone_number>",
|
| 1656 |
+
"single_word": false,
|
| 1657 |
+
"lstrip": false,
|
| 1658 |
+
"rstrip": false,
|
| 1659 |
+
"normalized": true,
|
| 1660 |
+
"special": false
|
| 1661 |
}
|
| 1662 |
],
|
| 1663 |
"normalizer": {
|
tokenizer_config.json
CHANGED
|
@@ -1359,6 +1359,118 @@
|
|
| 1359 |
"rstrip": false,
|
| 1360 |
"single_word": false,
|
| 1361 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1362 |
}
|
| 1363 |
},
|
| 1364 |
"additional_special_tokens": [
|
|
|
|
| 1359 |
"rstrip": false,
|
| 1360 |
"single_word": false,
|
| 1361 |
"special": false
|
| 1362 |
+
},
|
| 1363 |
+
"57687": {
|
| 1364 |
+
"content": "<s_retailer>",
|
| 1365 |
+
"lstrip": false,
|
| 1366 |
+
"normalized": true,
|
| 1367 |
+
"rstrip": false,
|
| 1368 |
+
"single_word": false,
|
| 1369 |
+
"special": false
|
| 1370 |
+
},
|
| 1371 |
+
"57688": {
|
| 1372 |
+
"content": "</s_retailer>",
|
| 1373 |
+
"lstrip": false,
|
| 1374 |
+
"normalized": true,
|
| 1375 |
+
"rstrip": false,
|
| 1376 |
+
"single_word": false,
|
| 1377 |
+
"special": false
|
| 1378 |
+
},
|
| 1379 |
+
"57689": {
|
| 1380 |
+
"content": "<s_retailer_name>",
|
| 1381 |
+
"lstrip": false,
|
| 1382 |
+
"normalized": true,
|
| 1383 |
+
"rstrip": false,
|
| 1384 |
+
"single_word": false,
|
| 1385 |
+
"special": false
|
| 1386 |
+
},
|
| 1387 |
+
"57690": {
|
| 1388 |
+
"content": "</s_retailer_name>",
|
| 1389 |
+
"lstrip": false,
|
| 1390 |
+
"normalized": true,
|
| 1391 |
+
"rstrip": false,
|
| 1392 |
+
"single_word": false,
|
| 1393 |
+
"special": false
|
| 1394 |
+
},
|
| 1395 |
+
"57691": {
|
| 1396 |
+
"content": "<s_retailer_address>",
|
| 1397 |
+
"lstrip": false,
|
| 1398 |
+
"normalized": true,
|
| 1399 |
+
"rstrip": false,
|
| 1400 |
+
"single_word": false,
|
| 1401 |
+
"special": false
|
| 1402 |
+
},
|
| 1403 |
+
"57692": {
|
| 1404 |
+
"content": "</s_retailer_address>",
|
| 1405 |
+
"lstrip": false,
|
| 1406 |
+
"normalized": true,
|
| 1407 |
+
"rstrip": false,
|
| 1408 |
+
"single_word": false,
|
| 1409 |
+
"special": false
|
| 1410 |
+
},
|
| 1411 |
+
"57693": {
|
| 1412 |
+
"content": "<s_retailer_pin>",
|
| 1413 |
+
"lstrip": false,
|
| 1414 |
+
"normalized": true,
|
| 1415 |
+
"rstrip": false,
|
| 1416 |
+
"single_word": false,
|
| 1417 |
+
"special": false
|
| 1418 |
+
},
|
| 1419 |
+
"57694": {
|
| 1420 |
+
"content": "</s_retailer_pin>",
|
| 1421 |
+
"lstrip": false,
|
| 1422 |
+
"normalized": true,
|
| 1423 |
+
"rstrip": false,
|
| 1424 |
+
"single_word": false,
|
| 1425 |
+
"special": false
|
| 1426 |
+
},
|
| 1427 |
+
"57695": {
|
| 1428 |
+
"content": "<s_retailer_phone_number>",
|
| 1429 |
+
"lstrip": false,
|
| 1430 |
+
"normalized": true,
|
| 1431 |
+
"rstrip": false,
|
| 1432 |
+
"single_word": false,
|
| 1433 |
+
"special": false
|
| 1434 |
+
},
|
| 1435 |
+
"57696": {
|
| 1436 |
+
"content": "</s_retailer_phone_number>",
|
| 1437 |
+
"lstrip": false,
|
| 1438 |
+
"normalized": true,
|
| 1439 |
+
"rstrip": false,
|
| 1440 |
+
"single_word": false,
|
| 1441 |
+
"special": false
|
| 1442 |
+
},
|
| 1443 |
+
"57697": {
|
| 1444 |
+
"content": "<s_supplier_id>",
|
| 1445 |
+
"lstrip": false,
|
| 1446 |
+
"normalized": true,
|
| 1447 |
+
"rstrip": false,
|
| 1448 |
+
"single_word": false,
|
| 1449 |
+
"special": false
|
| 1450 |
+
},
|
| 1451 |
+
"57698": {
|
| 1452 |
+
"content": "</s_supplier_id>",
|
| 1453 |
+
"lstrip": false,
|
| 1454 |
+
"normalized": true,
|
| 1455 |
+
"rstrip": false,
|
| 1456 |
+
"single_word": false,
|
| 1457 |
+
"special": false
|
| 1458 |
+
},
|
| 1459 |
+
"57699": {
|
| 1460 |
+
"content": "<s_supplier_phone_number>",
|
| 1461 |
+
"lstrip": false,
|
| 1462 |
+
"normalized": true,
|
| 1463 |
+
"rstrip": false,
|
| 1464 |
+
"single_word": false,
|
| 1465 |
+
"special": false
|
| 1466 |
+
},
|
| 1467 |
+
"57700": {
|
| 1468 |
+
"content": "</s_supplier_phone_number>",
|
| 1469 |
+
"lstrip": false,
|
| 1470 |
+
"normalized": true,
|
| 1471 |
+
"rstrip": false,
|
| 1472 |
+
"single_word": false,
|
| 1473 |
+
"special": false
|
| 1474 |
}
|
| 1475 |
},
|
| 1476 |
"additional_special_tokens": [
|