cache diagnostics

scripts/train.py

@@ -23,7 +23,20 @@ cache_dir = os.path.expanduser("~/.cache/huggingface/datasets")
 if os.path.exists(cache_dir):
     shutil.rmtree(cache_dir)
 
+#
+import datasets
+import os
+
+# Check current cache directory
+print("Current cache directory:", datasets.config.HF_DATASETS_CACHE)
+print("Cache exists:", os.path.exists(datasets.config.HF_DATASETS_CACHE))
 
+# List contents if it exists
+if os.path.exists(datasets.config.HF_DATASETS_CACHE):
+    print("Cache contents:")
+    for item in os.listdir(datasets.config.HF_DATASETS_CACHE):
+        print(f"  {item}")
+#
 def main():
     print("Starting Training Script for my cool OLMoE")
     # Load config - first try from local file, then from pretrained