set new dataset in train_tokenizer

config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": "./",
   "architectures": [
     "RobertaForMaskedLM"
   ],
@@ -18,6 +19,7 @@
   "num_hidden_layers": 12,
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
   "transformers_version": "4.9.0.dev0",
   "type_vocab_size": 1,
   "use_cache": true,

events.out.tfevents.1625996487.t1v-n-6a2ff29b-w-0.1982849.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac60639757fe4b60f9e4a84623140d1dae79d4ded1896534e2ae43a2a58e404d
+size 10516780

train_tokenizer.py

@@ -2,7 +2,8 @@ from datasets import load_dataset
 from tokenizers import trainers, Tokenizer, normalizers, ByteLevelBPETokenizer
 
 # load dataset
-dataset = load_dataset("mc4", "sw", split="train")
+# dataset = load_dataset("mc4", "sw", split="train")
+dataset = load_dataset("text", "sw", split="train", data_files={"train": ["/home/shared/clean_swahili/train.txt"]})
 
 # Instantiate tokenizer
 tokenizer = ByteLevelBPETokenizer()