WIP

script.py

@@ -40,10 +40,31 @@ if __name__ == "__main__":
     print(params)
     import os
     
-    print(os.system('pwd'))
+    print('pwd:', os.system('pwd'))
     print(os.system('ls -lahtr'))
+    print(os.system('ls -lahtr /tmp/data/'))
+    
+
+    data_path_test_server = Path('/tmp/data')
+    data_path_local = Path().home() / '.cache/huggingface/datasets/usm3d___hoho25k_test_x/'
+
+    if data_path_test_server.exists():
+        data_path = data_path_test_server
+    else: 
+        data_path = data_path_local
+    
+    print([str(p) for p in data_path.rglob('*validation*.arrow')])
 
-    dataset = load_dataset(params['dataset'], trust_remote_code=True, use_auth_token=params['token'])
+    # dataset = load_dataset(params['dataset'], trust_remote_code=True, use_auth_token=params['token'])
+    dataset = load_dataset(
+        "arrow", 
+        data_files={
+            "validation": [str(p) for p in data_path.rglob('*validation*.arrow')],
+            "test": [str(p) for p in data_path.rglob('*test*.arrow')],
+        },
+        trust_remote_code=True,
+        # streaming=True
+    )
     print(dataset, flush=True)
     # dataset = load_dataset('webdataset', data_files={)