added the tokenizer

README.md

@@ -1,4 +1,22 @@
 ---
+widget:
+- text: Acht Acht B.V.
+- text: Stichting Helison
+- text: Hayman
+- text: Melkveebedrijf Leferink
+- text: Jointly
+- text: Priority Shipping C.V.
+- text: Hoofd en Hart Bewindvoering
+- text: Dekker Kids Beheer N.V.
+- text: PCI H. Bartholomeus
+- text: AEGON Equity Europe Beta Pool (N)
+- text: ScaleX Ventures Fund I Coöperatief U.A.
+- text: Global Emerging Markets Equity Pool
+- text: Gemeente Groningen
+- text: Onderlinge Verzekeringmaatschappij "SOM" U.A.
+- text: Vereniging van Appartementseigenaars "Henry Dunantpark III"
+- text: Ferrovial Construction International SE
+- text: Rederij BigLift Barentsz
 model-index:
 - name: Sociovestix/lenu_NL
   results:
@@ -20,24 +38,6 @@ model-index:
       name: f1 macro
       args:
         average: macro
-widget:
-- text: "Acht Acht B.V."
-- text: "Stichting Helison"
-- text: "Hayman"
-- text: "Melkveebedrijf Leferink"
-- text: "Jointly"
-- text: "Priority Shipping C.V."
-- text: "Hoofd en Hart Bewindvoering"
-- text: "Dekker Kids Beheer N.V."
-- text: "PCI H. Bartholomeus"
-- text: "AEGON Equity Europe Beta Pool (N)"
-- text: "ScaleX Ventures Fund I Coöperatief U.A."
-- text: "Global Emerging Markets Equity Pool"
-- text: "Gemeente Groningen"
-- text: 'Onderlinge Verzekeringmaatschappij "SOM" U.A.'
-- text: 'Vereniging van Appartementseigenaars "Henry Dunantpark III"'
-- text: "Ferrovial Construction International SE"
-- text: "Rederij BigLift Barentsz"
 ---
 
 # LENU - Legal Entity Name Understanding for Netherlands

tokenizer_config.json

@@ -1,14 +1,53 @@
 {
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
-  "do_lower_case": false,
+  "do_lower_case": true,
   "mask_token": "[MASK]",
   "model_max_length": 512,
-  "name_or_path": "GroNLP/bert-base-dutch-cased",
-  "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "special_tokens_map_file": "/root/.cache/huggingface/hub/models--GroNLP--bert-base-dutch-cased/snapshots/484ff5cec2ad42b434537dadd901d9b8e2b64cd2/special_tokens_map.json",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",