waceke commited on
Commit
6bc0d67
·
verified ·
1 Parent(s): 734e871

Training done

Browse files
Files changed (3) hide show
  1. added_tokens.json +4 -0
  2. tokenizer.json +52 -2
  3. tokenizer_config.json +32 -0
added_tokens.json CHANGED
@@ -62,7 +62,9 @@
62
  "</s_shipper_email>": 57544,
63
  "</s_shipper_name>": 57538,
64
  "</s_shipper_no>": 57702,
 
65
  "</s_shipper_phone_no>": 57542,
 
66
  "</s_shipper_pin>": 57546,
67
  "</s_shipping_address>": 57649,
68
  "</s_shipping_date>": 57586,
@@ -153,7 +155,9 @@
153
  "<s_shipper_email>": 57543,
154
  "<s_shipper_name>": 57537,
155
  "<s_shipper_no>": 57701,
 
156
  "<s_shipper_phone_no>": 57541,
 
157
  "<s_shipper_pin>": 57545,
158
  "<s_shipping_address>": 57648,
159
  "<s_shipping_date>": 57585,
 
62
  "</s_shipper_email>": 57544,
63
  "</s_shipper_name>": 57538,
64
  "</s_shipper_no>": 57702,
65
+ "</s_shipper_number>": 57704,
66
  "</s_shipper_phone_no>": 57542,
67
+ "</s_shipper_phone_number>": 57706,
68
  "</s_shipper_pin>": 57546,
69
  "</s_shipping_address>": 57649,
70
  "</s_shipping_date>": 57586,
 
155
  "<s_shipper_email>": 57543,
156
  "<s_shipper_name>": 57537,
157
  "<s_shipper_no>": 57701,
158
+ "<s_shipper_number>": 57703,
159
  "<s_shipper_phone_no>": 57541,
160
+ "<s_shipper_phone_number>": 57705,
161
  "<s_shipper_pin>": 57545,
162
  "<s_shipping_address>": 57648,
163
  "<s_shipping_date>": 57585,
tokenizer.json CHANGED
@@ -1,7 +1,21 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
4
- "padding": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
@@ -1676,6 +1690,42 @@
1676
  "rstrip": false,
1677
  "normalized": true,
1678
  "special": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1679
  }
1680
  ],
1681
  "normalizer": {
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 768,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": {
10
+ "strategy": {
11
+ "Fixed": 768
12
+ },
13
+ "direction": "Right",
14
+ "pad_to_multiple_of": null,
15
+ "pad_id": 1,
16
+ "pad_type_id": 0,
17
+ "pad_token": "<pad>"
18
+ },
19
  "added_tokens": [
20
  {
21
  "id": 0,
 
1690
  "rstrip": false,
1691
  "normalized": true,
1692
  "special": false
1693
+ },
1694
+ {
1695
+ "id": 57703,
1696
+ "content": "<s_shipper_number>",
1697
+ "single_word": false,
1698
+ "lstrip": false,
1699
+ "rstrip": false,
1700
+ "normalized": true,
1701
+ "special": false
1702
+ },
1703
+ {
1704
+ "id": 57704,
1705
+ "content": "</s_shipper_number>",
1706
+ "single_word": false,
1707
+ "lstrip": false,
1708
+ "rstrip": false,
1709
+ "normalized": true,
1710
+ "special": false
1711
+ },
1712
+ {
1713
+ "id": 57705,
1714
+ "content": "<s_shipper_phone_number>",
1715
+ "single_word": false,
1716
+ "lstrip": false,
1717
+ "rstrip": false,
1718
+ "normalized": true,
1719
+ "special": false
1720
+ },
1721
+ {
1722
+ "id": 57706,
1723
+ "content": "</s_shipper_phone_number>",
1724
+ "single_word": false,
1725
+ "lstrip": false,
1726
+ "rstrip": false,
1727
+ "normalized": true,
1728
+ "special": false
1729
  }
1730
  ],
1731
  "normalizer": {
tokenizer_config.json CHANGED
@@ -1487,6 +1487,38 @@
1487
  "rstrip": false,
1488
  "single_word": false,
1489
  "special": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1490
  }
1491
  },
1492
  "additional_special_tokens": [
 
1487
  "rstrip": false,
1488
  "single_word": false,
1489
  "special": false
1490
+ },
1491
+ "57703": {
1492
+ "content": "<s_shipper_number>",
1493
+ "lstrip": false,
1494
+ "normalized": true,
1495
+ "rstrip": false,
1496
+ "single_word": false,
1497
+ "special": false
1498
+ },
1499
+ "57704": {
1500
+ "content": "</s_shipper_number>",
1501
+ "lstrip": false,
1502
+ "normalized": true,
1503
+ "rstrip": false,
1504
+ "single_word": false,
1505
+ "special": false
1506
+ },
1507
+ "57705": {
1508
+ "content": "<s_shipper_phone_number>",
1509
+ "lstrip": false,
1510
+ "normalized": true,
1511
+ "rstrip": false,
1512
+ "single_word": false,
1513
+ "special": false
1514
+ },
1515
+ "57706": {
1516
+ "content": "</s_shipper_phone_number>",
1517
+ "lstrip": false,
1518
+ "normalized": true,
1519
+ "rstrip": false,
1520
+ "single_word": false,
1521
+ "special": false
1522
  }
1523
  },
1524
  "additional_special_tokens": [