WIP

script.py

@@ -44,46 +44,66 @@ if __name__ == "__main__":
     os.system('pwd')
     print(os.system('ls -lahtr'))
     print('/tmp/data/')
-    print(os.system('ls -lahtrR /tmp/data/'))
+    print(os.system('ls -lahtr /tmp/data/'))
+    print('/tmp/data/data')
+    print(os.system('ls -lahtrR /tmp/data/data'))
     
 
     data_path_test_server = Path('/tmp/data')
     data_path_local = Path().home() / '.cache/huggingface/datasets/usm3d___hoho25k_test_x/'
 
     if data_path_test_server.exists():
-        data_path = data_path_test_server
+        # data_path = data_path_test_server
         TEST_ENV = True
     else: 
-        data_path = data_path_local
+        # data_path = data_path_local
         TEST_ENV = False
-
+        from huggingface_hub import snapshot_download 
+        _ = snapshot_download(
+            repo_id=params['dataset'],
+            local_dir="/tmp/data",
+            # token=params.token,
+            repo_type="dataset",
+        )
+    data_path = data_path_test_server
+    
     
     print(data_path)
 
     # dataset = load_dataset(params['dataset'], trust_remote_code=True, use_auth_token=params['token'])
+    # data_files = {
+    #     "validation": [str(p) for p in [*data_path.rglob('*validation*.arrow')]+[*data_path.rglob('*public*/**/*.tar')]],
+    #     "test": [str(p) for p in [*data_path.rglob('*test*.arrow')]+[*data_path.rglob('*private*/**/*.tar')]],
+    # }
     data_files = {
-        "validation": [str(p) for p in [*data_path.rglob('*validation*.arrow')]+[*data_path.rglob('*public*/**/*.tar')]],
-        "test": [str(p) for p in [*data_path.rglob('*test*.arrow')]+[*data_path.rglob('*private*/**/*.tar')]],
+        "validation": [str(p) for p in data_path.rglob('*public*/**/*.tar')],
+        "test": [str(p) for p in data_path.rglob('*private*/**/*.tar')],
     }
     print(data_files)
+    dataset = load_dataset(
+        params['dataset'],
+        data_files=data_files,
+        trust_remote_code=True,
+        # streaming=True
+    )
 
-
-    if TEST_ENV:
-        dataset = load_dataset(
-            "webdataset", 
-            data_files=data_files,
-            trust_remote_code=True,
-            # streaming=True
-        )
-        print('load with webdataset')
-    else:
-        dataset = load_dataset(
-            "arrow", 
-            data_files=data_files,
-            trust_remote_code=True,
-            # streaming=True
-        )
-        print('load with arrow')
+    # if TEST_ENV:
+    # dataset = load_dataset(
+    #     "webdataset", 
+    #     data_files=data_files,
+    #     trust_remote_code=True,
+    #     # streaming=True
+    # )
+    print('load with webdataset')
+    # else:
+        
+    #     dataset = load_dataset(
+    #         "arrow", 
+    #         data_files=data_files,
+    #         trust_remote_code=True,
+    #         # streaming=True
+    #     )
+    #     print('load with arrow')
     
 
     print(dataset, flush=True)