added the tokenizer

README.md

@@ -1,4 +1,23 @@
 ---
+widget:
+- text: CRH Group Services Ltd Share Participation Scheme
+- text: SCF Ajoneuvohallinto IX Limited
+- text: BSSF II Origination Designated Activity Company
+- text: New Vision Strategies ICAV
+- text: GPB EUROBOND FINANCE PUBLIC LIMITED COMPANY
+- text: IPF MANAGEMENT UNLIMITED COMPANY
+- text: Castlerea and District Credit Union Limited
+- text: IRPF Fernbank Limited Partnership
+- text: Disabled Drivers Association of Ireland
+- text: The Pat Brophy Family Partnership
+- text: PRIMA ICG SENIOR DEBT PARTNERS IRELAND DESIGNATED ACTIVITY COMPANY
+- text: East Coast Catering (Ireland) Unlimited
+- text: Marist Brothers In Ireland-Tallaght
+- text: Forestay Capital II, ILP
+- text: DONOVAN'S PHARMACY LIMITED
+- text: AXIS Specialty Europe SE
+- text: Ballycora Investments UC
+- text: Lakeland Dairies Co-Operative Society Ltd
 model-index:
 - name: Sociovestix/lenu_IE
   results:
@@ -20,25 +39,6 @@ model-index:
       name: f1 macro
       args:
         average: macro
-widget:
-- text: "CRH Group Services Ltd Share Participation Scheme"
-- text: "SCF Ajoneuvohallinto IX Limited"
-- text: "BSSF II Origination Designated Activity Company"
-- text: "New Vision Strategies ICAV"
-- text: "GPB EUROBOND FINANCE PUBLIC LIMITED COMPANY"
-- text: "IPF MANAGEMENT UNLIMITED COMPANY"
-- text: "Castlerea and District Credit Union Limited"
-- text: "IRPF Fernbank Limited Partnership"
-- text: "Disabled Drivers Association of Ireland"
-- text: "The Pat Brophy Family Partnership"
-- text: "PRIMA ICG SENIOR DEBT PARTNERS IRELAND DESIGNATED ACTIVITY COMPANY"
-- text: "East Coast Catering (Ireland) Unlimited"
-- text: "Marist Brothers In Ireland-Tallaght"
-- text: "Forestay Capital II, ILP"
-- text: "DONOVAN'S PHARMACY LIMITED"
-- text: "AXIS Specialty Europe SE"
-- text: "Ballycora Investments UC"
-- text: "Lakeland Dairies Co-Operative Society Ltd"
 ---
 
 # LENU - Legal Entity Name Understanding for Ireland

tokenizer_config.json

@@ -1,14 +1,53 @@
 {
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
   "do_lower_case": true,
   "mask_token": "[MASK]",
-  "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "yiyanghkust/finbert-pretrain",
-  "never_split": null,
+  "model_max_length": 512,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "special_tokens_map_file": null,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",