Enhanced dataset upload and rebuild logic to ensure datasets exist and are private before uploading. Updated relevant API calls to create private datasets.

scripts/utils/hf_persistence.py

@@ -68,6 +68,16 @@ class HFPersistenceManager:
         """
         try:
             api = self._get_api()
+            
+            # Ensure dataset exists and is private before uploading
+            try:
+                api.repo_info(repo_id=self.dataset_id, repo_type='dataset')
+                print(f'✓ Dataset exists: {self.dataset_id}')
+            except Exception:
+                print(f'📝 Dataset does not exist, creating private dataset: {self.dataset_id}')
+                api.create_repo(repo_id=self.dataset_id, repo_type='dataset', private=True)
+                print(f'✓ Private dataset created: {self.dataset_id}')
+            
             api.upload_file(
                 path_or_fileobj=local_path,
                 path_in_repo=remote_path,
@@ -156,7 +166,7 @@ class HFPersistenceManager:
                 
                 # Recreate dataset
                 print('🔨 Recreating dataset...')
-                api.create_repo(repo_id=self.dataset_id, repo_type='dataset', exist_ok=True)
+                api.create_repo(repo_id=self.dataset_id, repo_type='dataset', exist_ok=True, private=True)
                 print('✓ Dataset recreated successfully')
                 
                 # Restore backed up files
@@ -202,7 +212,7 @@ class HFPersistenceManager:
                 
                 # Recreate dataset
                 print('🔨 Recreating dataset...')
-                api.create_repo(repo_id=self.dataset_id, repo_type='dataset', exist_ok=True)
+                api.create_repo(repo_id=self.dataset_id, repo_type='dataset', exist_ok=True, private=True)
                 print('✓ Dataset recreated successfully')
                 
                 print('🎉 Dataset recreation and LFS cleanup completed!')