WIP

script.py

@@ -56,13 +56,14 @@ if __name__ == "__main__":
         data_path = data_path_local
     
     print(data_path)
-    print([str(p) for p in data_path.rglob('*validation*.(arrow|tar)')])
 
     # dataset = load_dataset(params['dataset'], trust_remote_code=True, use_auth_token=params['token'])
     data_files = {
         "validation": [str(p) for p in [*data_path.rglob('*validation*.arrow')]+[*data_path.rglob('*validation*.tar')]],
         "test": [str(p) for p in [*data_path.rglob('*test*.arrow')]+[*data_path.rglob('*test*.tar')]],
     }
+    print(data_files)
+
     try:
         dataset = load_dataset(
             "arrow",