Update 3 files

- /trainer.py
- /dataset.py
- /trainer.cli.py

dataset.py

@@ -0,0 +1,15 @@
+from datasets import load_dataset
+
+from init import Config
+
+
+class Dataset:
+    def __init__(self, config: Config):
+        self.__dict__ = dict(config.__dict__)
+
+        self.dataset = load_dataset(self.remote_path)
+        self.text = ''.join(s for s in self.dataset['train']['text']).encode('ascii', 'ignore').decode('ascii')
+
+
+    def Batch(self, ids):
+        pass

trainer.cli.py

@@ -23,5 +23,8 @@ if __name__ == '__main__':
 
     config = ConfigParser(args.config_path).config
     
+    dataset = Dataset(config.dataset)
+
+    #tokenizer = Tokenizer()
 
     trainer = Trainer(config)

trainer.py

@@ -8,12 +8,7 @@ class Trainer:
     def __init__(self, config: Config):
         self.__dict__ = dict(config.__dict__)
 
-        #print(f"self.dict: {self.__dict__}")
-        #print(f"locals: {locals()}")
-
         #self.wandb = Wandb(config.wandb)
         
-        #print(f"model config: {self.model}")
-        #print(f"config.params: {self.model.params}")
         self.model = Model(config.model)